You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ka...@apache.org on 2012/10/23 19:54:45 UTC
svn commit: r1401367 - in /oozie/trunk: ./
client/src/main/java/org/apache/oozie/client/
core/src/main/java/org/apache/oozie/command/wf/
core/src/test/java/org/apache/oozie/command/wf/
Author: kamrul
Date: Tue Oct 23 17:54:45 2012
New Revision: 1401367
URL: http://svn.apache.org/viewvc?rev=1401367&view=rev
Log:
OOZIE-1027 Command line mr does not support NN/JT parameters properly (Mona via Mohammad)
Modified:
oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java
oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java
oozie/trunk/release-log.txt
Modified: oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java
URL: http://svn.apache.org/viewvc/oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java?rev=1401367&r1=1401366&r2=1401367&view=diff
==============================================================================
--- oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java (original)
+++ oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java Tue Oct 23 17:54:45 2012
@@ -33,12 +33,14 @@ import org.json.simple.JSONValue;
public class XOozieClient extends OozieClient {
public static final String JT = "mapred.job.tracker";
+ public static final String JT_2 = "mapreduce.jobtracker.address";
public static final String NN = "fs.default.name";
-
+ public static final String NN_2 = "fs.defaultFS";
+
@Deprecated
public static final String JT_PRINCIPAL = "mapreduce.jobtracker.kerberos.principal";
-
+
@Deprecated
public static final String NN_PRINCIPAL = "dfs.namenode.kerberos.principal";
@@ -100,13 +102,19 @@ public class XOozieClient extends OozieC
private void validateHttpSubmitConf(Properties conf) {
String JT = conf.getProperty(XOozieClient.JT);
+ String JT_2 = conf.getProperty(XOozieClient.JT_2);
if (JT == null) {
- throw new RuntimeException("jobtracker is not specified in conf");
+ if(JT_2 == null) {
+ throw new RuntimeException("jobtracker is not specified in conf");
+ }
}
String NN = conf.getProperty(XOozieClient.NN);
+ String NN_2 = conf.getProperty(XOozieClient.NN_2);
if (NN == null) {
- throw new RuntimeException("namenode is not specified in conf");
+ if(NN_2 == null) {
+ throw new RuntimeException("namenode is not specified in conf");
+ }
}
String libPath = conf.getProperty(LIBPATH);
Modified: oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java?rev=1401367&r1=1401366&r2=1401367&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java (original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java Tue Oct 23 17:54:45 2012
@@ -25,6 +25,7 @@ import org.jdom.Namespace;
import org.apache.oozie.client.XOozieClient;
import org.apache.oozie.command.CommandException;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
@@ -32,6 +33,7 @@ import java.util.Set;
public class SubmitMRXCommand extends SubmitHttpXCommand {
private static final Set<String> SKIPPED_CONFS = new HashSet<String>();
+ private static final Map<String, String> DEPRECATE_MAP = new HashMap<String, String>();
public SubmitMRXCommand(Configuration conf, String authToken) {
super("submitMR", "submitMR", conf, authToken);
@@ -43,7 +45,11 @@ public class SubmitMRXCommand extends Su
SKIPPED_CONFS.add(XOozieClient.NN);
// a brillant mind made a change in Configuration that 'fs.default.name' key gets converted to 'fs.defaultFS'
// in Hadoop 0.23, we need skip that one too, keeping the old one because of Hadoop 1
- SKIPPED_CONFS.add("fs.defaultFS");
+ SKIPPED_CONFS.add(XOozieClient.NN_2);
+
+ DEPRECATE_MAP.put(XOozieClient.NN, XOozieClient.NN_2);
+ DEPRECATE_MAP.put(XOozieClient.JT, XOozieClient.JT_2);
+ DEPRECATE_MAP.put(WorkflowAppService.HADOOP_USER, "mapreduce.job.user.name");
}
private Element generateConfigurationSection(Configuration conf, Namespace ns) {
@@ -53,7 +59,8 @@ public class SubmitMRXCommand extends Su
Map.Entry<String, String> entry = iter.next();
String name = entry.getKey();
if (MANDATORY_OOZIE_CONFS.contains(name) || OPTIONAL_OOZIE_CONFS.contains(name)
- || SKIPPED_CONFS.contains(name)) {
+ || SKIPPED_CONFS.contains(name)
+ || DEPRECATE_MAP.containsValue(name)) {
continue;
}
@@ -78,10 +85,12 @@ public class SubmitMRXCommand extends Su
private Element generateMRSection(Configuration conf, Namespace ns) {
Element mapreduce = new Element("map-reduce", ns);
Element jt = new Element("job-tracker", ns);
- jt.addContent(conf.get(XOozieClient.JT));
+ String newJTVal = conf.get(DEPRECATE_MAP.get(XOozieClient.JT));
+ jt.addContent(newJTVal != null ? newJTVal : (conf.get(XOozieClient.JT)));
mapreduce.addContent(jt);
Element nn = new Element("name-node", ns);
- nn.addContent(conf.get(XOozieClient.NN));
+ String newNNVal = conf.get(DEPRECATE_MAP.get(XOozieClient.NN));
+ nn.addContent(newNNVal != null ? newNNVal : (conf.get(XOozieClient.NN)));
mapreduce.addContent(nn);
if (conf.size() > MANDATORY_OOZIE_CONFS.size()) { // excluding JT, NN,
@@ -113,8 +122,15 @@ public class SubmitMRXCommand extends Su
protected String getWorkflowXml(Configuration conf) {
for (String key : MANDATORY_OOZIE_CONFS) {
String value = conf.get(key);
- if (value == null) {
- throw new RuntimeException(key + " is not specified");
+ if(value == null) {
+ if(DEPRECATE_MAP.containsKey(key)) {
+ if(conf.get(DEPRECATE_MAP.get(key)) == null) {
+ throw new RuntimeException(key + " or " + DEPRECATE_MAP.get(key) + " is not specified");
+ }
+ }
+ else {
+ throw new RuntimeException(key + " is not specified");
+ }
}
}
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java?rev=1401367&r1=1401366&r2=1401367&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java Tue Oct 23 17:54:45 2012
@@ -113,4 +113,36 @@ public class TestSubmitMRXCommand extend
}
}
+
+ public void testWFXmlGenerationNewConfigProps() throws Exception {
+ try {
+ Configuration conf = new Configuration(false);
+ conf.set(XOozieClient.NN_2, "new_NN");
+ conf.set(XOozieClient.JT_2, "new_JT");
+ conf.set("mapred.mapper.class", "TestMapper");
+ conf.set("mapred.reducer.class", "TestReducer");
+ conf.set("mapred.input.dir", "testInput");
+ conf.set("mapred.output.dir", "testOutput");
+ conf.set(OozieClient.LIBPATH, "libpath");
+ conf.set("mapreduce.job.user.name", "test_user");
+
+ SubmitMRXCommand submitMRCmd = new SubmitMRXCommand(conf, "token");
+ String xml = submitMRCmd.getWorkflowXml(conf);
+
+ //verifying is a valid WF
+ WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
+ wps.parseDef(xml, conf);
+
+ Element wfE = XmlUtils.parseXml(xml);
+ Namespace ns = wfE.getNamespace();
+ Element actionE = wfE.getChild("action", ns).getChild("map-reduce", ns);
+ Element nnE = actionE.getChild("name-node", ns);
+ assertEquals(nnE.getTextTrim(), "new_NN");
+ Element jtE = actionE.getChild("job-tracker", ns);
+ assertEquals(jtE.getTextTrim(), "new_JT");
+ }
+ catch(Exception e) {
+ fail("should have passed");
+ }
+ }
}
Modified: oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1401367&r1=1401366&r2=1401367&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Tue Oct 23 17:54:45 2012
@@ -1,5 +1,6 @@
-- Oozie 3.4.0 release (trunk - unreleased)
+OOZIE-1027 Command line mr does not support NN/JT parameters properly (Mona via Mohammad)
OOZIE-1020 BulkJPAExecutor handling date-time value incorrectly.(Mona via Mohammad)`
OOZIE-967 Coordinator action window in web UI never finishes refreshing (kinley via tucu)
OOZIE-1024 ooziedb.sh script should keep OOZIE_CONFIG, OOZIE_LOG, and OOZIE_DATA if previously set (rkanter via tucu)