You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2016/01/15 19:39:50 UTC
oozie git commit: OOZIE-2439 FS Action no longer uses name-node from
global section or default NN (rkanter)
Repository: oozie
Updated Branches:
refs/heads/master ee8710bde -> 0cf2fb3e7
OOZIE-2439 FS Action no longer uses name-node from global section or default NN (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0cf2fb3e
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0cf2fb3e
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0cf2fb3e
Branch: refs/heads/master
Commit: 0cf2fb3e78fcc46d30270fcd86248b023a90c871
Parents: ee8710b
Author: Robert Kanter <rk...@cloudera.com>
Authored: Fri Jan 15 10:39:43 2016 -0800
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Fri Jan 15 10:39:43 2016 -0800
----------------------------------------------------------------------
.../oozie/action/hadoop/FsActionExecutor.java | 7 +-
.../workflow/lite/LiteWorkflowAppParser.java | 14 +++-
.../lite/TestLiteWorkflowAppParser.java | 85 ++++++++++++++++++++
.../wf-schema-fs-no-namenode-global.xml | 47 +++++++++++
.../test/resources/wf-schema-fs-no-namenode.xml | 44 ++++++++++
release-log.txt | 1 +
6 files changed, 196 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/0cf2fb3e/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
index b8649ac..121cd49 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
@@ -58,10 +58,12 @@ import org.jdom.Element;
*/
public class FsActionExecutor extends ActionExecutor {
+ public static final String ACTION_TYPE = "fs";
+
private final int maxGlobCount;
public FsActionExecutor() {
- super("fs");
+ super(ACTION_TYPE);
maxGlobCount = ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX);
}
@@ -645,4 +647,7 @@ public class FsActionExecutor extends ActionExecutor {
}
}
+ public boolean supportsConfigurationJobXML() {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/0cf2fb3e/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java b/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java
index 03c84f1..7f1c73c 100644
--- a/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java
+++ b/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java
@@ -20,6 +20,7 @@ package org.apache.oozie.workflow.lite;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.io.Writable;
+import org.apache.oozie.action.hadoop.FsActionExecutor;
import org.apache.oozie.action.oozie.SubWorkflowActionExecutor;
import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.util.ELUtils;
@@ -737,7 +738,13 @@ public class LiteWorkflowAppParser {
Namespace actionNs = actionElement.getNamespace();
+ // If this is the global section or ActionExecutor.requiresNameNodeJobTracker() returns true, we parse the action's
+ // <name-node> and <job-tracker> fields. If those aren't defined, we take them from the <global> section. If those
+ // aren't defined, we take them from the oozie-site defaults. If those aren't defined, we throw a WorkflowException.
+ // However, for the SubWorkflow and FS Actions, as well as the <global> section, we don't throw the WorkflowException.
+ // Also, we only parse the NN (not the JT) for the FS Action.
if (SubWorkflowActionExecutor.ACTION_TYPE.equals(actionElement.getName()) ||
+ FsActionExecutor.ACTION_TYPE.equals(actionElement.getName()) ||
GLOBAL.equals(actionElement.getName()) || ae.requiresNameNodeJobTracker()) {
if (actionElement.getChild(NAME_NODE, actionNs) == null) {
if (gData != null && gData.nameNode != null) {
@@ -745,11 +752,13 @@ public class LiteWorkflowAppParser {
} else if (defaultNameNode != null) {
addChildElement(actionElement, actionNs, NAME_NODE, defaultNameNode);
} else if (!(SubWorkflowActionExecutor.ACTION_TYPE.equals(actionElement.getName()) ||
+ FsActionExecutor.ACTION_TYPE.equals(actionElement.getName()) ||
GLOBAL.equals(actionElement.getName()))) {
throw new WorkflowException(ErrorCode.E0701, "No " + NAME_NODE + " defined");
}
}
- if (actionElement.getChild(JOB_TRACKER, actionNs) == null) {
+ if (actionElement.getChild(JOB_TRACKER, actionNs) == null &&
+ !FsActionExecutor.ACTION_TYPE.equals(actionElement.getName())) {
if (gData != null && gData.jobTracker != null) {
addChildElement(actionElement, actionNs, JOB_TRACKER, gData.jobTracker);
} else if (defaultJobTracker != null) {
@@ -761,6 +770,9 @@ public class LiteWorkflowAppParser {
}
}
+ // If this is the global section or ActionExecutor.supportsConfigurationJobXML() returns true, we parse the action's
+ // <configuration> and <job-xml> fields. We also merge this with those from the <global> section, if given. If none are
+ // defined, empty values are placed. Exceptions are thrown if there's an error parsing, but not if they're not given.
if ( GLOBAL.equals(actionElement.getName()) || ae.supportsConfigurationJobXML()) {
@SuppressWarnings("unchecked")
List<Element> actionJobXmls = actionElement.getChildren(JOB_XML, actionNs);
http://git-wip-us.apache.org/repos/asf/oozie/blob/0cf2fb3e/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
index 1fc1736..1983a8c 100644
--- a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
+++ b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
@@ -510,6 +510,91 @@ public class TestLiteWorkflowAppParser extends XTestCase {
}
}
+ public void testParserFsGlobalNN() throws Exception {
+ LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null,
+ LiteWorkflowStoreService.LiteControlNodeHandler.class,
+ LiteWorkflowStoreService.LiteDecisionHandler.class,
+ LiteWorkflowStoreService.LiteActionHandler.class);
+
+ LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-fs-no-namenode-global.xml", -1),
+ new Configuration());
+ String a = app.getNode("a").getConf();
+ String expectedA =
+ "<fs xmlns=\"uri:oozie:workflow:0.4\">\r\n" +
+ " <name-node>action-nn</name-node>\r\n" +
+ " <mkdir path=\"/foo\" />\r\n" +
+ " <configuration />\r\n" +
+ "</fs>";
+ a = cleanupXml(a);
+ assertEquals(expectedA.replaceAll(" ", ""), a.replaceAll(" ", ""));
+ String b = app.getNode("b").getConf();
+ String expectedB =
+ "<fs xmlns=\"uri:oozie:workflow:0.4\">\r\n" +
+ " <mkdir path=\"/foo\" />\r\n" +
+ " <name-node>global-nn</name-node>\r\n" +
+ " <configuration />\r\n" +
+ "</fs>";
+ b = cleanupXml(b);
+ assertEquals(expectedB.replaceAll(" ", ""), b.replaceAll(" ", ""));
+ }
+
+ public void testParserFsDefaultNN() throws Exception {
+ ConfigurationService.set("oozie.actions.default.name-node", "default-nn");
+ LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null,
+ LiteWorkflowStoreService.LiteControlNodeHandler.class,
+ LiteWorkflowStoreService.LiteDecisionHandler.class,
+ LiteWorkflowStoreService.LiteActionHandler.class);
+
+ LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-fs-no-namenode.xml", -1),
+ new Configuration());
+ String a = app.getNode("a").getConf();
+ String expectedA =
+ "<fs xmlns=\"uri:oozie:workflow:0.4\">\r\n" +
+ " <name-node>action-nn</name-node>\r\n" +
+ " <mkdir path=\"/foo\" />\r\n" +
+ " <configuration />\r\n" +
+ "</fs>";
+ a = cleanupXml(a);
+ assertEquals(expectedA.replaceAll(" ", ""), a.replaceAll(" ", ""));
+ String b = app.getNode("b").getConf();
+ String expectedB =
+ "<fs xmlns=\"uri:oozie:workflow:0.4\">\r\n" +
+ " <mkdir path=\"/foo\" />\r\n" +
+ " <name-node>default-nn</name-node>\r\n" +
+ " <configuration />\r\n" +
+ "</fs>";
+ b = cleanupXml(b);
+ assertEquals(expectedB.replaceAll(" ", ""), b.replaceAll(" ", ""));
+ }
+
+ public void testParserFsNoNN() throws Exception {
+ LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null,
+ LiteWorkflowStoreService.LiteControlNodeHandler.class,
+ LiteWorkflowStoreService.LiteDecisionHandler.class,
+ LiteWorkflowStoreService.LiteActionHandler.class);
+
+ LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-fs-no-namenode.xml", -1),
+ new Configuration());
+ String a = app.getNode("a").getConf();
+ String expectedA =
+ "<fs xmlns=\"uri:oozie:workflow:0.4\">\r\n" +
+ " <name-node>action-nn</name-node>\r\n" +
+ " <mkdir path=\"/foo\" />\r\n" +
+ " <configuration />\r\n" +
+ "</fs>";
+ a = cleanupXml(a);
+ assertEquals(expectedA.replaceAll(" ", ""), a.replaceAll(" ", ""));
+ // The FS Action shouldn't care if there's no NN in the end
+ String b = app.getNode("b").getConf();
+ String expectedB =
+ "<fs xmlns=\"uri:oozie:workflow:0.4\">\r\n" +
+ " <mkdir path=\"/foo\" />\r\n" +
+ " <configuration />\r\n" +
+ "</fs>";
+ b = cleanupXml(b);
+ assertEquals(expectedB.replaceAll(" ", ""), b.replaceAll(" ", ""));
+ }
+
public void testParser() throws Exception {
LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null,
LiteWorkflowStoreService.LiteControlNodeHandler.class,
http://git-wip-us.apache.org/repos/asf/oozie/blob/0cf2fb3e/core/src/test/resources/wf-schema-fs-no-namenode-global.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/wf-schema-fs-no-namenode-global.xml b/core/src/test/resources/wf-schema-fs-no-namenode-global.xml
new file mode 100644
index 0000000..d5afb58
--- /dev/null
+++ b/core/src/test/resources/wf-schema-fs-no-namenode-global.xml
@@ -0,0 +1,47 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<workflow-app xmlns="uri:oozie:workflow:0.4" name="test-wf">
+ <global>
+ <name-node>global-nn</name-node>
+ </global>
+
+ <start to="a"/>
+
+ <action name="a">
+ <fs>
+ <name-node>action-nn</name-node>
+ <mkdir path="/foo"/>
+ </fs>
+ <ok to="b"/>
+ <error to="c"/>
+ </action>
+
+ <action name="b">
+ <fs>
+ <mkdir path="/foo"/>
+ </fs>
+ <ok to="d"/>
+ <error to="c"/>
+ </action>
+
+ <kill name="c">
+ <message>fail</message>
+ </kill>
+
+ <end name="d"/>
+</workflow-app>
http://git-wip-us.apache.org/repos/asf/oozie/blob/0cf2fb3e/core/src/test/resources/wf-schema-fs-no-namenode.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/wf-schema-fs-no-namenode.xml b/core/src/test/resources/wf-schema-fs-no-namenode.xml
new file mode 100644
index 0000000..5b42b7c
--- /dev/null
+++ b/core/src/test/resources/wf-schema-fs-no-namenode.xml
@@ -0,0 +1,44 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<workflow-app xmlns="uri:oozie:workflow:0.4" name="test-wf">
+
+ <start to="a"/>
+
+ <action name="a">
+ <fs>
+ <name-node>action-nn</name-node>
+ <mkdir path="/foo"/>
+ </fs>
+ <ok to="b"/>
+ <error to="c"/>
+ </action>
+
+ <action name="b">
+ <fs>
+ <mkdir path="/foo"/>
+ </fs>
+ <ok to="d"/>
+ <error to="c"/>
+ </action>
+
+ <kill name="c">
+ <message>fail</message>
+ </kill>
+
+ <end name="d"/>
+</workflow-app>
http://git-wip-us.apache.org/repos/asf/oozie/blob/0cf2fb3e/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 1355d65..12a0641 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2439 FS Action no longer uses name-node from global section or default NN (rkanter)
OOZIE-2435 TestCoordChangeXCommand is flakey (fdenes via rkanter)
OOZIE-2428 TestSLAService, TestSLAEventGeneration flakey tests (fdenes via rkanter)
OOZIE-2437 Exclude additional Hadoop dependencies from Spark Sharelib (rkanter)