You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ja...@apache.org on 2016/09/13 12:09:16 UTC
oozie git commit: OOZIE-2583 oozie throws EL Exception when reference
variable name containing dot (abhishekbafna via jaydeepvishwakarma)
Repository: oozie
Updated Branches:
refs/heads/master abea46ac3 -> 7e12f0abc
OOZIE-2583 oozie throws EL Exception when reference variable name containing dot (abhishekbafna via jaydeepvishwakarma)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/7e12f0ab
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/7e12f0ab
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/7e12f0ab
Branch: refs/heads/master
Commit: 7e12f0abc69abe8873dd2d5f7ea4deaf617a5e04
Parents: abea46a
Author: jvishwakarma <jv...@walmartlabs.com>
Authored: Tue Sep 13 17:38:48 2016 +0530
Committer: jvishwakarma <jv...@walmartlabs.com>
Committed: Tue Sep 13 17:38:48 2016 +0530
----------------------------------------------------------------------
.../apache/oozie/command/wf/SubmitXCommand.java | 24 ++++++
.../oozie/command/wf/TestSubmitXCommand.java | 82 +++++++++++++++++++-
release-log.txt | 1 +
3 files changed, 105 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/7e12f0ab/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java
index 27274b9..3e2996f 100644
--- a/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java
@@ -160,6 +160,9 @@ public class SubmitXCommand extends WorkflowXCommand<String> {
throw new IOException("default configuration file, " + ex.getMessage(), ex);
}
}
+ if (defaultConf != null) {
+ defaultConf = resolveDefaultConfVariables(defaultConf);
+ }
WorkflowApp app = wps.parseDef(conf, defaultConf);
XConfiguration protoActionConf = wps.createProtoActionConf(conf, true);
@@ -278,6 +281,27 @@ public class SubmitXCommand extends WorkflowXCommand<String> {
}
}
+ /**
+ * Resolving variables from config-default, which might be referencing into conf/defaultConf
+ * @param defaultConf config-default.xml
+ * @return resolved config-default configuration.
+ */
+ private Configuration resolveDefaultConfVariables(Configuration defaultConf) {
+ XConfiguration resolveDefaultConf = new XConfiguration();
+ for (Map.Entry<String, String> entry : defaultConf) {
+ String defaultConfKey = entry.getKey();
+ String defaultConfValue = entry.getValue();
+ // if value is referencing some other key, first check within the default config to resolve,
+ // then job.properties (conf)
+ if (defaultConfValue.contains("$") && defaultConf.get(defaultConfKey).contains("$")) {
+ resolveDefaultConf.set(defaultConfKey, conf.get(defaultConfKey));
+ } else {
+ resolveDefaultConf.set(defaultConfKey, defaultConf.get(defaultConfKey));
+ }
+ }
+ return resolveDefaultConf;
+ }
+
private void removeSlaElements(Element eWfJob) {
Element sla = XmlUtils.getSLAElement(eWfJob);
if (sla != null) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/7e12f0ab/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
index 73464c8..3c893d0 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
@@ -19,14 +19,19 @@
package org.apache.oozie.command.wf;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.StringReader;
import java.net.URI;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.ErrorCode;
+import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
+import org.apache.oozie.executor.jpa.WorkflowActionQueryExecutor;
import org.apache.oozie.local.LocalOozie;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.WorkflowJob;
@@ -41,6 +46,8 @@ import org.apache.oozie.test.XDataTestCase;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.service.XLogService;
import org.apache.oozie.util.IOUtils;
+import org.apache.oozie.util.XmlUtils;
+import org.jdom.Element;
public class TestSubmitXCommand extends XDataTestCase {
@Override
@@ -260,7 +267,7 @@ public class TestSubmitXCommand extends XDataTestCase {
waitFor(15 * 1000, new Predicate() {
public boolean evaluate() throws Exception {
- return wfClient.getJobInfo(jobId).getStatus() == WorkflowJob.Status.SUCCEEDED;
+ return wfClient.getJobInfo(jobId).getStatus() == WorkflowJob.Status.PREP;
}
});
WorkflowJobBean wf = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW, jobId);
@@ -283,7 +290,7 @@ public class TestSubmitXCommand extends XDataTestCase {
waitFor(15 * 1000, new Predicate() {
public boolean evaluate() throws Exception {
- return wfClient.getJobInfo(jobId1).getStatus() == WorkflowJob.Status.SUCCEEDED;
+ return wfClient.getJobInfo(jobId1).getStatus() == WorkflowJob.Status.PREP;
}
});
wf = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW, jobId1);
@@ -297,6 +304,77 @@ public class TestSubmitXCommand extends XDataTestCase {
assertNotNull(protoConf.get(WorkflowAppService.APP_LIB_PATH_LIST));
}
+ public void testWFConfigDefaultVarResolve() throws Exception {
+ final OozieClient wfClient = LocalOozie.getClient();
+ OutputStream os = new FileOutputStream(getTestCaseDir() + "/config-default.xml");
+ XConfiguration defaultConf = new XConfiguration();
+ defaultConf.set("outputDir", "default-output-dir");
+ defaultConf.set("foo.bar", "default-foo-bar");
+ defaultConf.set("foobarRef", "${foo.bar}");
+ defaultConf.set("key", "default_value");
+ defaultConf.set("should_resolve", "${should.resolve}");
+ defaultConf.set("mixed", "${nameNode}/${outputDir}");
+ defaultConf.writeXml(os);
+ os.close();
+
+ String workflowUri = getTestCaseFileUri("workflow.xml");
+ String actionXml = "<map-reduce>"
+ + "<job-tracker>${jobTracker}</job-tracker>"
+ + "<name-node>${nameNode}</name-node>"
+ + " <prepare>"
+ + " <delete path=\"${nameNode}/user/${wf:user()}/mr/${outputDir}\"/>"
+ + " </prepare>"
+ + " <configuration>"
+ + " <property><name>bb</name><value>BB</value></property>"
+ + " <property><name>cc</name><value>from_action</value></property>"
+ + " </configuration>"
+ + " </map-reduce>";
+ String wfXml = "<workflow-app xmlns=\"uri:oozie:workflow:0.5\" name=\"map-reduce-wf\">"
+ + " <start to=\"mr-node\"/>"
+ + " <action name=\"mr-node\">"
+ + actionXml
+ + " <ok to=\"end\"/>"
+ + " <error to=\"fail\"/>"
+ + "</action>"
+ + "<kill name=\"fail\">"
+ + " <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>"
+ + "</kill>"
+ + "<end name=\"end\"/>"
+ + "</workflow-app>";
+
+ writeToFile(wfXml, workflowUri);
+ Configuration conf = new XConfiguration();
+ conf.set("nameNode", getNameNodeUri());
+ conf.set("jobTracker", getJobTrackerUri());
+ conf.set("foobarRef", "foobarRef");
+ conf.set("key", "job_prop_value");
+ conf.set(OozieClient.APP_PATH, workflowUri);
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ conf.set("should.resolve", "resolved");
+ SubmitXCommand sc = new SubmitXCommand(conf);
+ final String jobId = sc.call();
+ new StartXCommand(jobId).call();
+ waitFor(15 * 1000, new Predicate() {
+ public boolean evaluate() throws Exception {
+ return wfClient.getJobInfo(jobId).getStatus() == WorkflowJob.Status.KILLED;
+ }
+ });
+ String actionId = jobId + "@mr-node";
+ WorkflowActionBean action = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQueryExecutor.WorkflowActionQuery
+ .GET_ACTION, actionId);
+ Element eAction = XmlUtils.parseXml(action.getConf());
+ Element eConf = eAction.getChild("configuration", eAction.getNamespace());
+ Configuration actionConf = new XConfiguration(new StringReader(XmlUtils.prettyPrint(eConf).toString()));
+ assertEquals("default-output-dir", actionConf.get("outputDir"));
+ assertEquals("BB", actionConf.get("bb"));
+ assertEquals("from_action", actionConf.get("cc"));
+ assertEquals("resolved", actionConf.get("should_resolve"));
+ assertEquals("default-foo-bar", actionConf.get("foo.bar"));
+ assertEquals("default-foo-bar", actionConf.get("foobarRef"));
+ assertEquals("default_value", actionConf.get("key"));
+ assertEquals(getNameNodeUri()+"/default-output-dir", actionConf.get("mixed"));
+ }
+
private void writeToFile(String appXml, String appPath) throws IOException {
// TODO Auto-generated method stub
File wf = new File(URI.create(appPath));
http://git-wip-us.apache.org/repos/asf/oozie/blob/7e12f0ab/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 8e6e791..c03a5c7 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2583 oozie throws EL Exception when reference variable name containing dot (abhishekbafna via jaydeepvishwakarma)
OOZIE-2653 Close active connection to hcat server in fs action (satishsaley via puru)
OOZIE-2617 Read properties files in action configurations (wutaklon via jaydeepvishwakarma)
OOZIE-2615 Flaky tests TestCoordActionsKillXCommand.testActionKillCommandActionNumbers and testActionKillCommandDate (pbacsko via rkanter)