You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by tu...@apache.org on 2012/11/27 21:18:27 UTC
svn commit: r1414369 - in /oozie/trunk:
core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
core/src/main/resources/oozie-default.xml
core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
release-log.txt
Author: tucu
Date: Tue Nov 27 20:18:26 2012
New Revision: 1414369
URL: http://svn.apache.org/viewvc?rev=1414369&view=rev
Log:
OOZIE-1089 DistributedCache workaround for Hadoop 2.0.2-alpha (tucu)
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
oozie/trunk/core/src/main/resources/oozie-default.xml
oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
oozie/trunk/release-log.txt
Modified: oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java?rev=1414369&r1=1414368&r2=1414369&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java (original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java Tue Nov 27 20:18:26 2012
@@ -33,6 +33,9 @@ import java.lang.reflect.Method;
import java.net.URI;
import java.security.Permission;
import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.concurrent.ScheduledThreadPoolExecutor;
@@ -201,6 +204,18 @@ public class LauncherMapper<K1, V1, K2,
actionConf.set(OOZIE_JOB_ID, jobId);
actionConf.set(OOZIE_ACTION_ID, actionId);
+ if (Services.get().getConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)) {
+ List<String> purgedEntries = new ArrayList<String>();
+ Collection<String> entries = actionConf.getStringCollection("mapreduce.job.cache.files");
+ for (String entry : entries) {
+ if (entry.contains("#")) {
+ purgedEntries.add(entry);
+ }
+ }
+ actionConf.setStrings("mapreduce.job.cache.files", purgedEntries.toArray(new String[purgedEntries.size()]));
+ launcherConf.setBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", true);
+ }
+
FileSystem fs =
Services.get().get(HadoopAccessorService.class).createFileSystem(launcherConf.get("user.name"),
actionDir.toUri(), launcherConf);
@@ -398,6 +413,9 @@ public class LauncherMapper<K1, V1, K2,
}
else {
String mainClass = getJobConf().get(CONF_OOZIE_ACTION_MAIN_CLASS);
+ if (getJobConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)) {
+ System.err.println("WARNING, workaround for Hadoop 2.0.2-alpha distributed cached issue (MAPREDUCE-4820) enabled");
+ }
String msgPrefix = "Main class [" + mainClass + "], ";
int errorCode = 0;
Throwable errorCause = null;
Modified: oozie/trunk/core/src/main/resources/oozie-default.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/resources/oozie-default.xml?rev=1414369&r1=1414368&r2=1414369&view=diff
==============================================================================
--- oozie/trunk/core/src/main/resources/oozie-default.xml (original)
+++ oozie/trunk/core/src/main/resources/oozie-default.xml Tue Nov 27 20:18:26 2012
@@ -1679,4 +1679,18 @@
</property>
+ <!-- Enable Distributed Cache workaround for Hadoop 2.0.2-alpha (MAPREDUCE-4820) -->
+ <property>
+ <name>oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache</name>
+ <value>false</value>
+ <description>
+ Due to a bug in Hadoop 2.0.2-alpha, MAPREDUCE-4820, launcher jobs fail to set
+ the distributed cache for the action job because the local JARs are implicitly
+ included triggering a duplicate check.
+ This flag removes the distributed cache files for the action as they'll be
+ included from the local JARs of the JobClient (MRApps) submitting the action
+ job from the launcher.
+ </description>
+ </property>
+
</configuration>
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java?rev=1414369&r1=1414368&r2=1414369&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java Tue Nov 27 20:18:26 2012
@@ -294,4 +294,34 @@ public class TestLauncher extends XFsTes
lm.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0", actionConf, prepareBlock);
assertTrue(jobConf.get("oozie.action.prepare.xml").equals(prepareBlock));
}
+
+ // Test to ensure that the property value "oozie.action.prepare.xml" in the configuration of the job is properly set
+ // when there is prepare block in workflow XML
+ public void testSetupLauncherInfoHadoop2_0_2_alphaWorkaround() throws Exception {
+ Path actionDir = getFsTestCaseDir();
+ FileSystem fs = getFileSystem();
+ Path newDir = new Path(actionDir, "newDir");
+
+ // Setting up the job configuration
+ JobConf jobConf = Services.get().get(HadoopAccessorService.class).
+ createJobConf(new URI(getNameNodeUri()).getAuthority());
+ jobConf.set("user.name", getTestUser());
+ jobConf.set("fs.default.name", getNameNodeUri());
+
+ LauncherMapper lm = new LauncherMapper();
+ Configuration actionConf = new XConfiguration();
+ actionConf.set("mapreduce.job.cache.files", "a.jar,aa.jar#aa.jar");
+ lm.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0", actionConf, "");
+ assertFalse(jobConf.getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false));
+ assertEquals("a.jar,aa.jar#aa.jar", actionConf.get("mapreduce.job.cache.files"));
+
+ Services.get().getConf().setBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", true);
+ lm = new LauncherMapper();
+ actionConf = new XConfiguration();
+ actionConf.set("mapreduce.job.cache.files", "a.jar,aa.jar#aa.jar");
+ lm.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0", actionConf, "");
+ assertTrue(jobConf.getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false));
+ assertEquals("aa.jar#aa.jar", actionConf.get("mapreduce.job.cache.files"));
+ }
+
}
Modified: oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1414369&r1=1414368&r2=1414369&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Tue Nov 27 20:18:26 2012
@@ -50,6 +50,7 @@ OOZIE-944 Implement Workflow Generator U
-- Oozie 3.3.0 release (unreleased)
+OOZIE-1089 DistributedCache workaround for Hadoop 2.0.2-alpha (tucu)
OOZIE-1058 ACL modify-job should not be hardcoded to group name(mona via mohammad)
OOZIE-1052 HadoopAccessorService.createFileSystem throws exception in map-reduce action, failing workflow.(ryota via mohammad).
OOZIE-1060 bump hadoop 2.X version to 2.0.2-alpha (rvs via tucu)