You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ro...@apache.org on 2016/06/07 15:32:20 UTC
oozie git commit: OOZIE-2553 Cred tag is required for all actions in
the workflow even if an action does not require it (me.venkatr via rohini)
Repository: oozie
Updated Branches:
refs/heads/master fc9aeb13a -> f83f484a1
OOZIE-2553 Cred tag is required for all actions in the workflow even if an action does not require it (me.venkatr via rohini)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f83f484a
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f83f484a
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f83f484a
Branch: refs/heads/master
Commit: f83f484a1a34c9446cf43ba7d627aefabaf58921
Parents: fc9aeb1
Author: Rohini Palaniswamy <ro...@apache.org>
Authored: Tue Jun 7 08:32:13 2016 -0700
Committer: Rohini Palaniswamy <ro...@apache.org>
Committed: Tue Jun 7 08:32:13 2016 -0700
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 12 ++++---
.../org/apache/oozie/workflow/lite/NodeDef.java | 5 ++-
.../action/hadoop/ActionExecutorTestCase.java | 2 --
.../action/hadoop/TestJavaActionExecutor.java | 38 +++++++++++++++++++-
.../command/wf/TestActionCheckXCommand.java | 1 -
.../command/wf/TestActionStartXCommand.java | 4 ---
.../apache/oozie/event/TestEventGeneration.java | 1 -
.../oozie/service/TestRecoveryService.java | 1 -
.../apache/oozie/store/TestDBWorkflowStore.java | 1 -
.../org/apache/oozie/test/XDataTestCase.java | 1 -
release-log.txt | 1 +
11 files changed, 49 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 6893bb8..639003e 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -1307,11 +1307,13 @@ public class JavaActionExecutor extends ActionExecutor {
HashMap<String, CredentialsProperties> props = new HashMap<String, CredentialsProperties>();
if (context != null && action != null) {
String credsInAction = action.getCred();
- LOG.debug("Get credential '" + credsInAction + "' properties for action : " + action.getId());
- String[] credNames = credsInAction.split(",");
- for (String credName : credNames) {
- CredentialsProperties credProps = getCredProperties(context, credName);
- props.put(credName, credProps);
+ if (credsInAction != null) {
+ LOG.debug("Get credential '" + credsInAction + "' properties for action : " + action.getId());
+ String[] credNames = credsInAction.split(",");
+ for (String credName : credNames) {
+ CredentialsProperties credProps = getCredProperties(context, credName);
+ props.put(credName, credProps);
+ }
}
}
else {
http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java b/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java
index a395b77..9e66d28 100644
--- a/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java
+++ b/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java
@@ -40,7 +40,7 @@ public class NodeDef implements Writable {
private Class<? extends NodeHandler> handlerClass;
private String conf = null;
private List<String> transitions = new ArrayList<String>();
- private String cred = "null";
+ private String cred = null;
private String userRetryMax = "null";
private String userRetryInterval = "null";
@@ -154,6 +154,9 @@ public class NodeDef implements Writable {
nodeDefVersion = LiteWorkflowStoreService.NODE_DEF_VERSION_1;
name = dataInput.readUTF();
cred = dataInput.readUTF();
+ if (cred.equals("null")) {
+ cred = null;
+ }
String handlerClassName = dataInput.readUTF();
if ((handlerClassName != null) && (handlerClassName.length() > 0)) {
try {
http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java b/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java
index e1c450c..d74160a 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java
@@ -262,7 +262,6 @@ public abstract class ActionExecutorTestCase extends XHCatTestCase {
WorkflowActionBean action = new WorkflowActionBean();
action.setName(actionName);
- action.setCred("null");
action.setId(Services.get().get(UUIDService.class).generateChildId(workflow.getId(), actionName));
workflow.getActions().add(action);
return workflow;
@@ -296,7 +295,6 @@ public abstract class ActionExecutorTestCase extends XHCatTestCase {
WorkflowActionBean action = new WorkflowActionBean();
action.setName(actionName);
- action.setCred("null");
action.setId(Services.get().get(UUIDService.class).generateChildId(workflow.getId(), actionName));
workflow.getActions().add(action);
return workflow;
http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 057c9fb..85bb993 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -35,6 +35,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -1025,6 +1026,42 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
}
}
+
+ public void testCredentialsWithoutCredTag() throws Exception {
+ // create a workflow with credentials
+ // add a pig action without cred tag
+ String workflowXml = "<workflow-app xmlns='uri:oozie:workflow:0.2.5' name='pig-wf'>" + "<credentials>"
+ + "<credential name='abcname' type='abc'>" + "<property>" + "<name>property1</name>"
+ + "<value>value1</value>" + "</property>" + "<property>" + "<name>property2</name>"
+ + "<value>value2</value>" + "</property>" + "<property>" + "<name>${property3}</name>"
+ + "<value>${value3}</value>" + "</property>" + "</credential>" + "</credentials>"
+ + "<start to='pig1' />" + "<action name='pig1'>" + "<pig>" + "</pig>"
+ + "<ok to='end' />" + "<error to='fail' />" + "</action>" + "<kill name='fail'>"
+ + "<message>Pig failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>" + "</kill>"
+ + "<end name='end' />" + "</workflow-app>";
+
+ JavaActionExecutor ae = new JavaActionExecutor();
+ WorkflowJobBean wfBean = addRecordToWfJobTable("test1", workflowXml);
+ WorkflowActionBean action = (WorkflowActionBean) wfBean.getActions().get(0);
+ action.setType(ae.getType());
+ String actionXml = "<pig>" + "<job-tracker>${jobTracker}</job-tracker>" + "<name-node>${nameNode}</name-node>"
+ + "<prepare>" + "<delete path='outputdir' />" + "</prepare>" + "<configuration>" + "<property>"
+ + "<name>mapred.compress.map.output</name>" + "<value>true</value>" + "</property>" + "<property>"
+ + "<name>mapred.job.queue.name</name>" + "<value>${queueName}</value>" + "</property>"
+ + "</configuration>" + "<script>org/apache/oozie/examples/pig/id.pig</script>"
+ + "<param>INPUT=${inputDir}</param>" + "<param>OUTPUT=${outputDir}/pig-output</param>" + "</pig>";
+ action.setConf(actionXml);
+ Context context = new Context(wfBean, action);
+
+ Element actionXmlconf = XmlUtils.parseXml(action.getConf());
+ // action job configuration
+ Configuration actionConf = ae.createBaseHadoopConf(context, actionXmlconf);
+
+ // should not throw JA021 exception
+ HashMap<String, CredentialsProperties> credProperties = ae.setCredentialPropertyToActionConf(context, action,
+ actionConf);
+ }
+
public void testCredentialsSkip() throws Exception {
// Try setting oozie.credentials.skip at different levels, and verifying the correct behavior
// oozie-site: false -- job-level: null -- action-level: null
@@ -1163,7 +1200,6 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
wfBean.setStatus(WorkflowJob.Status.SUCCEEDED);
WorkflowActionBean action = new WorkflowActionBean();
action.setName("test");
- action.setCred("null");
action.setId(Services.get().get(UUIDService.class).generateChildId(wfBean.getId(), "test"));
wfBean.getActions().add(action);
return wfBean;
http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java
index f503b1f..5898d1a 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java
@@ -633,7 +633,6 @@ public class TestActionCheckXCommand extends XDataTestCase {
WorkflowActionBean action = new WorkflowActionBean();
String actionname = "testAction";
action.setName(actionname);
- action.setCred("null");
action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname));
action.setJobId(wfId);
action.setType("map-reduce");
http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java
index b7489e9..ea90c08 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java
@@ -334,7 +334,6 @@ public class TestActionStartXCommand extends XDataTestCase {
WorkflowActionBean action = new WorkflowActionBean();
String actionname = "testAction";
action.setName(actionname);
- action.setCred("null");
action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname));
action.setJobId(wfId);
action.setType("map-reduce");
@@ -421,7 +420,6 @@ public class TestActionStartXCommand extends XDataTestCase {
WorkflowActionBean action = new WorkflowActionBean();
String actionname = "testAction";
action.setName(actionname);
- action.setCred("null");
action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname));
action.setJobId(wfId);
action.setType("map-reduce");
@@ -529,7 +527,6 @@ public class TestActionStartXCommand extends XDataTestCase {
WorkflowActionBean action = new WorkflowActionBean();
String actionname = "testAction";
action.setName(actionname);
- action.setCred("null");
action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname));
action.setJobId(wfId);
action.setType("map-reduce");
@@ -568,7 +565,6 @@ public class TestActionStartXCommand extends XDataTestCase {
WorkflowActionBean action = new WorkflowActionBean();
String actionname = "testAction";
action.setName(actionname);
- action.setCred("null");
action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname));
action.setJobId(wfId);
action.setType("fs");
http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java b/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java
index f662d8a..afd3f8f 100644
--- a/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java
+++ b/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java
@@ -686,7 +686,6 @@ public class TestEventGeneration extends XDataTestCase {
action.setStartTime(new Date());
action.setEndTime(new Date());
action.setLastCheckTime(new Date());
- action.setCred("null");
action.setPendingOnly();
String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java b/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java
index 13d8e8d..eab177b 100644
--- a/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java
@@ -864,7 +864,6 @@ public class TestRecoveryService extends XDataTestCase {
WorkflowActionBean action = new WorkflowActionBean();
String actionname = "testAction";
action.setName(actionname);
- action.setCred("null");
action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname));
action.setJobId(wfId);
action.setType("map-reduce");
http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/store/TestDBWorkflowStore.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/store/TestDBWorkflowStore.java b/core/src/test/java/org/apache/oozie/store/TestDBWorkflowStore.java
index c263e86..b571fb5 100644
--- a/core/src/test/java/org/apache/oozie/store/TestDBWorkflowStore.java
+++ b/core/src/test/java/org/apache/oozie/store/TestDBWorkflowStore.java
@@ -477,7 +477,6 @@ public class TestDBWorkflowStore extends XTestCase {
a31.setId(str.toString());
a31.setJobId(wfBean3.getId());
a31.setStatus(WorkflowAction.Status.PREP);
- a31.setCred("null");
store.beginTrx();
store.insertAction(a31);
store.commitTrx();
http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/test/XDataTestCase.java b/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
index a9aa79a..081d2f1 100644
--- a/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
+++ b/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
@@ -1417,7 +1417,6 @@ public abstract class XDataTestCase extends XHCatTestCase {
action.setStartTime(currDate);
action.setEndTime(currDate);
action.setLastCheckTime(currDate);
- action.setCred("null");
action.setStats("dummyStats");
if (pending) {
action.setPending();
http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 189ca21..feea868 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2553 Cred tag is required for all actions in the workflow even if an action does not require it (me.venkatr via rohini)
OOZIE-2503 show ChildJobURLs to spark action (satishsaley via puru)
OOZIE-2551 Feature request: epoch timestamp generation (jtolar via puru)
OOZIE-2542 Option to disable OpenJPA BrokerImpl finalization (puru)