You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pu...@apache.org on 2016/05/10 16:49:13 UTC
oozie git commit: OOZIE-2471 Show child job url tab for distcp
Repository: oozie
Updated Branches:
refs/heads/master fafb5cf03 -> e9d3d3cb2
OOZIE-2471 Show child job url tab for distcp
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/e9d3d3cb
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/e9d3d3cb
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/e9d3d3cb
Branch: refs/heads/master
Commit: e9d3d3cb269ed954d38de0306929e4857d650301
Parents: fafb5cf
Author: Purshotam Shah <pu...@yahoo-inc.com>
Authored: Tue May 10 09:48:13 2016 -0700
Committer: Purshotam Shah <pu...@yahoo-inc.com>
Committed: Tue May 10 09:48:13 2016 -0700
----------------------------------------------------------------------
.../action/hadoop/DistcpActionExecutor.java | 19 ++++++
.../action/hadoop/TestDistCpActionExecutor.java | 18 +++++-
.../oozie/action/hadoop/TestDistcpMain.java | 10 +++
release-log.txt | 1 +
.../apache/oozie/action/hadoop/DistcpMain.java | 64 +++++++++++++++++++-
webapp/src/main/webapp/oozie-console.js | 2 +-
6 files changed, 111 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/e9d3d3cb/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
index 42f2965..96726da 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
@@ -18,16 +18,24 @@
package org.apache.oozie.action.hadoop;
+import java.io.IOException;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.RunningJob;
import org.apache.oozie.action.ActionExecutorException;
+import org.apache.oozie.action.ActionExecutor.Context;
+import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.service.ConfigurationService;
+import org.apache.oozie.service.HadoopAccessorException;
import org.apache.oozie.service.Services;
import org.apache.oozie.util.XLog;
import org.jdom.Element;
+import org.jdom.JDOMException;
public class DistcpActionExecutor extends JavaActionExecutor{
public static final String CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS = "org.apache.oozie.action.hadoop.DistcpMain";
@@ -117,4 +125,15 @@ public class DistcpActionExecutor extends JavaActionExecutor{
return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS);
}
+ @Override
+ protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context)
+ throws HadoopAccessorException, JDOMException, IOException, URISyntaxException {
+ super.getActionData(actionFs, runningJob, action, context);
+ readExternalChildIDs(action, context);
+ }
+
+ @Override
+ protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/e9d3d3cb/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java
index 939a332..e8a140f 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java
@@ -31,10 +31,12 @@ import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
+import org.apache.oozie.action.hadoop.ActionExecutorTestCase.Context;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.WorkflowAppService;
+import org.apache.oozie.test.XTestCase.Predicate;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XConfiguration;
@@ -96,8 +98,21 @@ public class TestDistCpActionExecutor extends ActionExecutorTestCase{
assertEquals(readContent[offset], content[offset]);
offset++;
}
- }
+ // Check for external ids
+ DistcpActionExecutor ae = new DistcpActionExecutor();
+ WorkflowAction wfAction = context.getAction();
+ ae.check(context, wfAction);
+ ae.end(context, wfAction);
+
+ assertEquals("SUCCEEDED", wfAction.getExternalStatus());
+ String externalIds = wfAction.getExternalChildIDs();
+ assertNotNull(externalIds);
+ assertNotSame("", externalIds);
+ // check for the expected prefix of hadoop jobIDs
+ assertTrue(externalIds.contains("job_"));
+
+ }
protected Context createContext(String actionXml) throws Exception {
DistcpActionExecutor ae = new DistcpActionExecutor();
@@ -148,4 +163,5 @@ public class TestDistCpActionExecutor extends ActionExecutorTestCase{
assertNotNull(runningJob);
return runningJob;
}
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/e9d3d3cb/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
index 9581c5f..10c0e1c 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
@@ -58,6 +58,16 @@ public class TestDistcpMain extends MainTestCase {
System.setProperty("oozie.action.conf.xml", actionXml.getAbsolutePath());
+ File statsDataFile = new File(getTestCaseDir(), "statsdata.properties");
+ File hadoopIdsFile = new File(getTestCaseDir(), "hadoopIds");
+ File outputDataFile = new File(getTestCaseDir(), "outputdata.properties");
+
+ setSystemProperty("oozie.launcher.job.id", "" + System.currentTimeMillis());
+ setSystemProperty("oozie.action.conf.xml", actionXml.getAbsolutePath());
+ setSystemProperty("oozie.action.stats.properties", statsDataFile.getAbsolutePath());
+ setSystemProperty("oozie.action.externalChildIDs", hadoopIdsFile.getAbsolutePath());
+ setSystemProperty("oozie.action.output.properties", outputDataFile.getAbsolutePath());
+
// Check normal execution
DistcpMain.main(new String[]{inputDir.toString(), outputDir.toString()});
assertTrue(getFileSystem().exists(outputDir));
http://git-wip-us.apache.org/repos/asf/oozie/blob/e9d3d3cb/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 67f48e8..e9aa963 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2471 Show child job url tab for distcp (satishsaley via puru)
OOZIE-2511 SubWorkflow missing variable set from option if config-default is present in parent workflow (asasvari via rkanter)
OOZIE-2391 spark-opts value in workflow.xml is not parsed properly (gezapeti via rkanter)
OOZIE-2489 XML parsing is vulnerable (fdenes via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/e9d3d3cb/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
----------------------------------------------------------------------
diff --git a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
index 325798f..6ac5ad6 100644
--- a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
+++ b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
@@ -18,18 +18,28 @@
package org.apache.oozie.action.hadoop;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
+import java.net.URL;
+import java.util.Properties;
+import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.PropertyConfigurator;
public class DistcpMain extends JavaMain {
private Constructor<?> construct;
private Object[] constArgs;
-
+ private static final String DISTCP_LOG4J_PROPS = "distcp-log4j.properties";
+ private static final Pattern[] DISTCP_JOB_IDS_PATTERNS = { Pattern.compile("Job complete: (job_\\S*)"),
+ Pattern.compile("Job (job_\\S*) completed successfully") };
public static void main(String[] args) throws Exception {
run(DistcpMain.class, args);
}
@@ -39,6 +49,7 @@ public class DistcpMain extends JavaMain {
Configuration actionConf = loadActionConf();
LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
+ String logFile = setUpDistcpLog4J(actionConf);
Class<?> klass = actionConf.getClass(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS,
org.apache.hadoop.tools.DistCp.class);
System.out.println("Main class : " + klass.getName());
@@ -69,6 +80,9 @@ public class DistcpMain extends JavaMain {
catch (InvocationTargetException ex) {
throw new JavaMainException(ex.getCause());
}
+ finally {
+ writeExternalChildIDs(logFile, DISTCP_JOB_IDS_PATTERNS, "Distcp");
+ }
}
protected void getConstructorAndArgs(Class<?> klass, Configuration actionConf) throws Exception {
@@ -95,4 +109,52 @@ public class DistcpMain extends JavaMain {
}
}
+ public static String setUpDistcpLog4J(Configuration distcpConf) throws IOException {
+ // Logfile to capture job IDs
+ String hadoopJobId = System.getProperty("oozie.launcher.job.id");
+ if (hadoopJobId == null) {
+ throw new RuntimeException("Launcher Hadoop Job ID system,property not set");
+ }
+
+ String logFile = new File("distcp-oozie-" + hadoopJobId + ".log").getAbsolutePath();
+
+ Properties hadoopProps = new Properties();
+
+ // Preparing log4j configuration
+ URL log4jFile = Thread.currentThread().getContextClassLoader().getResource("log4j.properties");
+ if (log4jFile != null) {
+ // getting hadoop log4j configuration
+ hadoopProps.load(log4jFile.openStream());
+ }
+
+ String logLevel = distcpConf.get("oozie.distcp.log.level", "INFO");
+ String rootLogLevel = distcpConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO");
+
+ hadoopProps.setProperty("log4j.rootLogger", rootLogLevel + ", A");
+ hadoopProps.setProperty("log4j.logger.org.apache.hadoop.tools", logLevel + ", A, jobid");
+ hadoopProps.setProperty("log4j.additivity.org.apache.hadoop.tools", "false");
+ hadoopProps.setProperty("log4j.appender.A", "org.apache.log4j.ConsoleAppender");
+ hadoopProps.setProperty("log4j.appender.A.layout", "org.apache.log4j.PatternLayout");
+ hadoopProps.setProperty("log4j.appender.A.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
+
+ hadoopProps.setProperty("log4j.appender.jobid", "org.apache.log4j.FileAppender");
+ hadoopProps.setProperty("log4j.appender.jobid.file", logFile);
+ hadoopProps.setProperty("log4j.appender.jobid.layout", "org.apache.log4j.PatternLayout");
+ hadoopProps.setProperty("log4j.appender.jobid.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
+ hadoopProps.setProperty("log4j.logger.org.apache.hadoop.mapred", "INFO, jobid");
+ hadoopProps.setProperty("log4j.logger.org.apache.hadoop.mapreduce.Job", "INFO, jobid");
+
+ String localProps = new File(DISTCP_LOG4J_PROPS).getAbsolutePath();
+ OutputStream os1 = new FileOutputStream(localProps);
+ try {
+ hadoopProps.store(os1, "");
+ }
+ finally {
+ os1.close();
+ }
+
+ PropertyConfigurator.configure(DISTCP_LOG4J_PROPS);
+
+ return logFile;
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/e9d3d3cb/webapp/src/main/webapp/oozie-console.js
----------------------------------------------------------------------
diff --git a/webapp/src/main/webapp/oozie-console.js b/webapp/src/main/webapp/oozie-console.js
index 31dcc3d..99dc6ce 100644
--- a/webapp/src/main/webapp/oozie-console.js
+++ b/webapp/src/main/webapp/oozie-console.js
@@ -765,7 +765,7 @@ function jobDetailsPopup(response, request) {
items : urlUnit
};
if (actionStatus.type == "pig" || actionStatus.type == "hive" || actionStatus.type == "map-reduce"
- || actionStatus.type == "hive2" || actionStatus.type == "sqoop") {
+ || actionStatus.type == "hive2" || actionStatus.type == "sqoop" || actionStatus.type == "distcp") {
var tabPanel = win.items.get(0);
tabPanel.add(childJobsItem);
}