You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ro...@apache.org on 2014/05/06 19:16:52 UTC
git commit: OOZIE-1806 Java Action type jobs are failing with
hadoop-0.20.0 and earlier versions on oozie trunk (sathish.mittal via rohini)
Repository: oozie
Updated Branches:
refs/heads/master 9c91c814c -> 6e2e46a98
OOZIE-1806 Java Action type jobs are failing with hadoop-0.20.0 and earlier versions on oozie trunk (sathish.mittal via rohini)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/6e2e46a9
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/6e2e46a9
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/6e2e46a9
Branch: refs/heads/master
Commit: 6e2e46a98fa59377134ec0f5303ead1b6f1c7532
Parents: 9c91c81
Author: Rohini Palaniswamy <ro...@yahoo-inc.com>
Authored: Tue May 6 10:16:45 2014 -0700
Committer: Rohini Palaniswamy <ro...@yahoo-inc.com>
Committed: Tue May 6 10:16:45 2014 -0700
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 7 +++--
.../java/org/apache/oozie/util/JobUtils.java | 30 ++++++++++++++++++++
release-log.txt | 1 +
3 files changed, 35 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/6e2e46a9/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index d4b4f5e..40add2c 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -66,6 +66,7 @@ import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.servlet.CallbackServlet;
import org.apache.oozie.util.ELEvaluator;
+import org.apache.oozie.util.JobUtils;
import org.apache.oozie.util.LogUtils;
import org.apache.oozie.util.PropertiesUtils;
import org.apache.oozie.util.XConfiguration;
@@ -524,7 +525,7 @@ public class JavaActionExecutor extends ActionExecutor {
if (listOfPaths != null && !listOfPaths.isEmpty()) {
for (Path actionLibPath : listOfPaths) {
- DistributedCache.addFileToClassPath(actionLibPath, conf, fs);
+ JobUtils.addFileToClassPath(actionLibPath, conf, fs);
DistributedCache.createSymlink(conf);
}
}
@@ -554,13 +555,13 @@ public class JavaActionExecutor extends ActionExecutor {
}
FileSystem fs = listOfPaths.get(0).getFileSystem(conf);
for (Path actionLibPath : listOfPaths) {
- DistributedCache.addFileToClassPath(actionLibPath, conf, fs);
+ JobUtils.addFileToClassPath(actionLibPath, conf, fs);
DistributedCache.createSymlink(conf);
}
listOfPaths = shareLibService.getSystemLibJars(getType());
if (listOfPaths != null) {
for (Path actionLibPath : listOfPaths) {
- DistributedCache.addFileToClassPath(actionLibPath, conf, fs);
+ JobUtils.addFileToClassPath(actionLibPath, conf, fs);
DistributedCache.createSymlink(conf);
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/6e2e46a9/core/src/main/java/org/apache/oozie/util/JobUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/JobUtils.java b/core/src/main/java/org/apache/oozie/util/JobUtils.java
index 25eb735..135b096 100644
--- a/core/src/main/java/org/apache/oozie/util/JobUtils.java
+++ b/core/src/main/java/org/apache/oozie/util/JobUtils.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.oozie.ErrorCode;
@@ -133,4 +134,33 @@ public class JobUtils {
return map;
}
+
+ /**
+ * This method provides a wrapper around hadoop 0.20/1.x and 0.23/2.x implementations.
+ * TODO: Remove the workaround when we drop the support for hadoop 0.20.
+ * @param file Path of the file to be added
+ * @param conf Configuration that contains the classpath setting
+ * @param fs FileSystem with respect to which path should be interpreted
+ * @throws IOException
+ */
+ public static void addFileToClassPath(Path file, Configuration conf, FileSystem fs) throws IOException {
+ Configuration defaultConf = new Configuration();
+ XConfiguration.copy(conf, defaultConf);
+ DistributedCache.addFileToClassPath(file, defaultConf, fs);
+ // Hadoop 0.20/1.x.
+ if (defaultConf.get("mapred.job.classpath.files") != null) {
+ // Duplicate hadoop 1.x code to workaround MAPREDUCE-2361 in Hadoop 0.20
+ // Refer OOZIE-1806.
+ String filepath = file.toUri().getPath();
+ String classpath = conf.get("mapred.job.classpath.files");
+ conf.set("mapred.job.classpath.files", classpath == null
+ ? filepath
+ : classpath + System.getProperty("path.separator") + filepath);
+ URI uri = fs.makeQualified(file).toUri();
+ DistributedCache.addCacheFile(uri, conf);
+ }
+ else { // Hadoop 0.23/2.x
+ DistributedCache.addFileToClassPath(file, conf, fs);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/6e2e46a9/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index b2fcf6d..795311a 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1806 Java Action type jobs are failing with hadoop-0.20.0 and earlier versions on oozie trunk (sathish.mittal via rohini)
OOZIE-1823 OozieSharelibCLI shouldn't load ext services (rkanter)
OOZIE-1762 Sharelib with oozie.action.ship.launcher.jar=true should copy oozie-hadoop-utils.jar (puru via mona)
OOZIE-1709 CoordELFunctions.getCurrentInstance() is expensive (shwethags via rohini)