You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pb...@apache.org on 2017/05/18 14:58:28 UTC
oozie git commit: Address code review comments
Repository: oozie
Updated Branches:
refs/heads/oya 5dd08b159 -> be32cda04
Address code review comments
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/be32cda0
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/be32cda0
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/be32cda0
Branch: refs/heads/oya
Commit: be32cda04577b46fe4df292f859214b739e5a39b
Parents: 5dd08b1
Author: Peter Bacsko <pb...@cloudera.com>
Authored: Thu May 18 16:58:18 2017 +0200
Committer: Peter Bacsko <pb...@cloudera.com>
Committed: Thu May 18 16:58:18 2017 +0200
----------------------------------------------------------------------
.../oozie/action/hadoop/DistcpActionExecutor.java | 2 +-
.../oozie/action/hadoop/HiveActionExecutor.java | 2 +-
.../oozie/action/hadoop/JavaActionExecutor.java | 14 ++------------
.../oozie/action/hadoop/MapReduceActionExecutor.java | 2 +-
.../apache/oozie/action/hadoop/PigActionExecutor.java | 2 +-
.../oozie/action/hadoop/ShellActionExecutor.java | 8 ++++++++
.../oozie/action/hadoop/SparkActionExecutor.java | 2 +-
.../oozie/action/hadoop/SqoopActionExecutor.java | 2 +-
.../oozie/action/hadoop/TestJavaActionExecutor.java | 6 ++++++
.../oozie/action/hadoop/TestShellActionExecutor.java | 2 +-
.../test/java/org/apache/oozie/test/XTestCase.java | 2 +-
sharelib/oozie/pom.xml | 1 +
12 files changed, 25 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
index fb26399..20f47d5 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
@@ -107,7 +107,7 @@ public class DistcpActionExecutor extends JavaActionExecutor{
}
@Override
- protected boolean needToAddMRJars() {
+ protected boolean needToAddMapReduceToClassPath() {
return true;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
index 012d1ec..22378fc 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
@@ -132,7 +132,7 @@ public class HiveActionExecutor extends ScriptLanguageActionExecutor {
}
@Override
- protected boolean needToAddMRJars() {
+ protected boolean needToAddMapReduceToClassPath() {
return true;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 87deb69..c879c90 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -314,14 +314,6 @@ public class JavaActionExecutor extends ActionExecutor {
}
}
- void updateConfForJavaTmpDir(Configuration conf) {
- String amChildOpts = conf.get(YARN_AM_COMMAND_OPTS);
- String oozieJavaTmpDirSetting = "-Djava.io.tmpdir=./tmp";
- if (amChildOpts != null && !amChildOpts.contains(JAVA_TMP_DIR_SETTINGS)) {
- conf.set(YARN_AM_COMMAND_OPTS, amChildOpts + " " + oozieJavaTmpDirSetting);
- }
- }
-
public static void parseJobXmlAndConfiguration(Context context, Element element, Path appPath, Configuration conf)
throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException {
parseJobXmlAndConfiguration(context, element, appPath, conf, false);
@@ -402,7 +394,6 @@ public class JavaActionExecutor extends ActionExecutor {
// set cancel.delegation.token in actionConf that child job doesn't cancel delegation token
actionConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false);
- updateConfForJavaTmpDir(actionConf);
setRootLoggerLevel(actionConf);
return actionConf;
}
@@ -862,7 +853,6 @@ public class JavaActionExecutor extends ActionExecutor {
launcherJobConf.set(HADOOP_CHILD_JAVA_OPTS, opts.toString().trim());
launcherJobConf.set(HADOOP_MAP_JAVA_OPTS, opts.toString().trim());
- updateConfForJavaTmpDir(launcherJobConf);
injectLauncherTimelineServiceEnabled(launcherJobConf, actionConf);
// properties from action that are needed by the launcher (e.g. QUEUE NAME, ACLs)
@@ -1085,7 +1075,7 @@ public class JavaActionExecutor extends ActionExecutor {
// This adds the Hadoop jars to the classpath in the Launcher JVM
ClasspathUtils.setupClasspath(env, launcherJobConf);
- if (needToAddMRJars()) {
+ if (needToAddMapReduceToClassPath()) {
ClasspathUtils.addMapReduceToClasspath(env, launcherJobConf);
}
@@ -1343,7 +1333,7 @@ public class JavaActionExecutor extends ActionExecutor {
* MR jars to the classpath.
* @return false by default
*/
- protected boolean needToAddMRJars() {
+ protected boolean needToAddMapReduceToClassPath() {
return false;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
index 2050591..ca7c7f2 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
@@ -342,7 +342,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor {
}
@Override
- protected boolean needToAddMRJars() {
+ protected boolean needToAddMapReduceToClassPath() {
return true;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
index 17508eb..8a24ac3 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
@@ -123,7 +123,7 @@ public class PigActionExecutor extends ScriptLanguageActionExecutor {
}
@Override
- protected boolean needToAddMRJars() {
+ protected boolean needToAddMapReduceToClassPath() {
return true;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
index 9d71b78..63b8722 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
@@ -19,10 +19,13 @@
package org.apache.oozie.action.hadoop;
+import java.io.File;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.util.Apps;
import org.apache.oozie.action.ActionExecutorException;
import org.apache.oozie.service.ConfigurationService;
import org.jdom.Element;
@@ -148,6 +151,11 @@ public class ShellActionExecutor extends JavaActionExecutor {
updateProperty(conf, OOZIE_LAUNCHER_CHILD_ENV, envValues);
}
+ @Override
+ protected void addActionSpecificEnvVars(Map<String, String> env) {
+ Apps.setEnvFromInputString(env, "PATH=.:$PATH", File.pathSeparator);
+ }
+
/**
* Utility method to append the new value to any property.
*
http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
index 2161af0..00497a7 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
@@ -143,7 +143,7 @@ public class SparkActionExecutor extends JavaActionExecutor {
}
@Override
- protected boolean needToAddMRJars() {
+ protected boolean needToAddMapReduceToClassPath() {
return true;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
index 73cc190..955f3b7 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
@@ -237,7 +237,7 @@ public class SqoopActionExecutor extends JavaActionExecutor {
}
@Override
- protected boolean needToAddMRJars() {
+ protected boolean needToAddMapReduceToClassPath() {
return true;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 972cc1c..f14f060 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -31,11 +31,14 @@ import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.examples.SleepJob;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -43,8 +46,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.action.ActionExecutor;
@@ -58,6 +63,7 @@ import org.apache.oozie.service.LiteWorkflowStoreService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.ShareLibService;
import org.apache.oozie.service.UUIDService;
+import org.apache.oozie.service.UserGroupInformationService;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.service.WorkflowStoreService;
import org.apache.oozie.util.IOUtils;
http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
index 1c3b33e..72be0a2 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
@@ -42,7 +42,7 @@ public class TestShellActionExecutor extends ActionExecutorTestCase {
private static final String SHELL_EXEC = Shell.WINDOWS ? "cmd.exe" : "sh";
private static final String SHELL_PARAM = Shell.WINDOWS ? "/c" : "-c";
- private static final String SHELL_SCRIPTNAME = Shell.WINDOWS ? "script.cmd" : "./script.sh";
+ private static final String SHELL_SCRIPTNAME = Shell.WINDOWS ? "script.cmd" : "script.sh";
private static final String SHELL_SCRIPT_CONTENT = Shell.WINDOWS
? "dir /s /b\necho %1 %2\necho %PATH%\ntype %0"
: "ls -ltr\necho $1 $2\necho $PATH\npwd\ntype sh";
http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/core/src/test/java/org/apache/oozie/test/XTestCase.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java
index 72d646d..d7ce9b2 100644
--- a/core/src/test/java/org/apache/oozie/test/XTestCase.java
+++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java
@@ -1235,7 +1235,7 @@ public abstract class XTestCase extends TestCase {
final ApplicationId appId = ConverterUtils.toApplicationId(externalId);
final MutableObject<YarnApplicationState> finalState = new MutableObject<YarnApplicationState>();
- Configuration conf = Services.get().get(HadoopAccessorService.class).getCachedConf();
+ Configuration conf = Services.get().get(HadoopAccessorService.class).createJobConf(getJobTrackerUri());
final YarnClient yarnClient = Services.get().get(HadoopAccessorService.class).createYarnClient(getTestUser(), conf);
try {
http://git-wip-us.apache.org/repos/asf/oozie/blob/be32cda0/sharelib/oozie/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/oozie/pom.xml b/sharelib/oozie/pom.xml
index 0ccca1d..12f5cdd 100644
--- a/sharelib/oozie/pom.xml
+++ b/sharelib/oozie/pom.xml
@@ -64,6 +64,7 @@
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>