You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2015/01/23 23:18:20 UTC
oozie git commit: OOZIE-1722 Add support to Hadoop-2 for AM restarts
of the launcher job (jaydeepvishwakarma via rkanter)
Repository: oozie
Updated Branches:
refs/heads/master d5f88032a -> bdd4914fa
OOZIE-1722 Add support to Hadoop-2 for AM restarts of the launcher job (jaydeepvishwakarma via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/bdd4914f
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/bdd4914f
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/bdd4914f
Branch: refs/heads/master
Commit: bdd4914fa95808bcc4192b5ccc8464554153ecd7
Parents: d5f8803
Author: Robert Kanter <rk...@cloudera.com>
Authored: Fri Jan 23 14:13:02 2015 -0800
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Fri Jan 23 14:13:02 2015 -0800
----------------------------------------------------------------------
.../action/hadoop/LauncherMainHadoopUtils.java | 79 +++++++++++++++++++-
release-log.txt | 1 +
2 files changed, 77 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/bdd4914f/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
index 46c2fbd..f6bb6a4 100644
--- a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
+++ b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
@@ -18,19 +18,92 @@
package org.apache.oozie.action.hadoop;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
-
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.mapreduce.TypeConverter;
public class LauncherMainHadoopUtils {
private LauncherMainHadoopUtils() {
}
+ private static Set<ApplicationId> getChildYarnJobs(Configuration actionConf) {
+ Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>();
+ long startTime = 0L;
+ try {
+ startTime = Long.parseLong((System.getProperty("oozie.job.launch.time")));
+ } catch(NumberFormatException nfe) {
+ throw new RuntimeException("Could not find Oozie job launch time", nfe);
+ }
+ String tag = actionConf.get("mapreduce.job.tags");
+ if (tag == null) {
+ throw new RuntimeException("Could not find Yarn tags property (mapreduce.job.tags)");
+ }
+ GetApplicationsRequest gar = GetApplicationsRequest.newInstance();
+ gar.setScope(ApplicationsRequestScope.OWN);
+ gar.setStartRange(startTime, System.currentTimeMillis());
+ gar.setApplicationTags(Collections.singleton(tag));
+ try {
+ ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(actionConf, ApplicationClientProtocol.class);
+ GetApplicationsResponse apps = proxy.getApplications(gar);
+ List<ApplicationReport> appsList = apps.getApplicationList();
+ for(ApplicationReport appReport : appsList) {
+ childYarnJobs.add(appReport.getApplicationId());
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException("Exception occurred while finding child jobs", ioe);
+ } catch (YarnException ye) {
+ throw new RuntimeException("Exception occurred while finding child jobs", ye);
+ }
+ return childYarnJobs;
+ }
+
public static String getYarnJobForMapReduceAction(Configuration actionConf) {
- return null;
+ Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf);
+ String childJobId = null;
+ if (!childYarnJobs.isEmpty()) {
+ ApplicationId childJobYarnId = childYarnJobs.iterator().next();
+ System.out.println("Found Map-Reduce job [" + childJobYarnId + "] already running");
+ // Need the JobID version for Oozie
+ childJobId = TypeConverter.fromYarn(childJobYarnId).toString();
+ }
+ return childJobId;
}
public static void killChildYarnJobs(Configuration actionConf) {
- // no-op
+ try {
+ Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf);
+ if (!childYarnJobs.isEmpty()) {
+ System.out.println();
+ System.out.println("Found [" + childYarnJobs.size() + "] Map-Reduce jobs from this launcher");
+ System.out.println("Killing existing jobs and starting over:");
+ YarnClient yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(actionConf);
+ yarnClient.start();
+ for (ApplicationId app : childYarnJobs) {
+ System.out.print("Killing job [" + app + "] ... ");
+ yarnClient.killApplication(app);
+ System.out.println("Done");
+ }
+ System.out.println();
+ }
+ } catch (YarnException ye) {
+ throw new RuntimeException("Exception occurred while killing child job(s)", ye);
+ } catch (IOException ioe) {
+ throw new RuntimeException("Exception occurred while killing child job(s)", ioe);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/bdd4914f/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 6dda7b1..b834cbc 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.2.0 release (trunk - unreleased)
+OOZIE-1722 Add support to Hadoop-2 for AM restarts of the launcher job (jaydeepvishwakarma via rkanter)
OOZIE-2107 Schema config properties should be consistent with ActionExecutor config properties (rkanter)
OOZIE-1730 Change hadoop-2 profile to use 2.4.0 (jaydeepvishwakarma via rkanter)
OOZIE-2088 Exponential retries for transient failures (pavan kumar via shwethags)