You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2015/09/09 19:33:17 UTC
oozie git commit: OOZIE-2356 Add a way to enable/disable credentials
in a workflow (rkanter)
Repository: oozie
Updated Branches:
refs/heads/master 396fcc6c4 -> 1502633e7
OOZIE-2356 Add a way to enable/disable credentials in a workflow (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/1502633e
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/1502633e
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/1502633e
Branch: refs/heads/master
Commit: 1502633e76c63c2da6ac055284d0b0eb4c4dc44a
Parents: 396fcc6
Author: Robert Kanter <rk...@cloudera.com>
Authored: Wed Sep 9 10:33:04 2015 -0700
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Wed Sep 9 10:33:04 2015 -0700
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 65 ++++++----
core/src/main/resources/oozie-default.xml | 8 ++
.../action/hadoop/TestJavaActionExecutor.java | 124 +++++++++++++++++++
.../site/twiki/DG_ActionAuthentication.twiki | 13 ++
release-log.txt | 1 +
5 files changed, 188 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/1502633e/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 0ea2eea..a6e7000 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -122,6 +122,7 @@ public class JavaActionExecutor extends ActionExecutor {
public static final String CONF_HADOOP_YARN_UBER_MODE = "oozie.action.launcher." + HADOOP_YARN_UBER_MODE;
public static final String HADOOP_JOB_CLASSLOADER = "mapreduce.job.classloader";
public static final String HADOOP_USER_CLASSPATH_FIRST = "mapreduce.user.classpath.first";
+ public static final String OOZIE_CREDENTIALS_SKIP = "oozie.credentials.skip";
static {
DISALLOWED_PROPERTIES.add(HADOOP_USER);
@@ -1052,18 +1053,21 @@ public class JavaActionExecutor extends ActionExecutor {
}
// Setting the credential properties in launcher conf
+ JobConf credentialsConf = null;
HashMap<String, CredentialsProperties> credentialsProperties = setCredentialPropertyToActionConf(context,
action, actionConf);
+ if (credentialsProperties != null) {
- // Adding if action need to set more credential tokens
- JobConf credentialsConf = new JobConf(false);
- XConfiguration.copy(actionConf, credentialsConf);
- setCredentialTokens(credentialsConf, context, action, credentialsProperties);
+ // Adding if action need to set more credential tokens
+ credentialsConf = new JobConf(false);
+ XConfiguration.copy(actionConf, credentialsConf);
+ setCredentialTokens(credentialsConf, context, action, credentialsProperties);
- // insert conf to action conf from credentialsConf
- for (Entry<String, String> entry : credentialsConf) {
- if (actionConf.get(entry.getKey()) == null) {
- actionConf.set(entry.getKey(), entry.getValue());
+ // insert conf to action conf from credentialsConf
+ for (Entry<String, String> entry : credentialsConf) {
+ if (actionConf.get(entry.getKey()) == null) {
+ actionConf.set(entry.getKey(), entry.getValue());
+ }
}
}
@@ -1098,7 +1102,7 @@ public class JavaActionExecutor extends ActionExecutor {
launcherJobConf.getCredentials().addToken(HadoopAccessorService.MR_TOKEN_ALIAS, mrdt);
// insert credentials tokens to launcher job conf if needed
- if (needInjectCredentials()) {
+ if (needInjectCredentials() && credentialsConf != null) {
for (Token<? extends TokenIdentifier> tk : credentialsConf.getCredentials().getAllTokens()) {
Text fauxAlias = new Text(tk.getKind() + "_" + tk.getService());
LOG.debug("ADDING TOKEN: " + fauxAlias);
@@ -1163,24 +1167,39 @@ public class JavaActionExecutor extends ActionExecutor {
WorkflowAction action, Configuration actionConf) throws Exception {
HashMap<String, CredentialsProperties> credPropertiesMap = null;
if (context != null && action != null) {
- credPropertiesMap = getActionCredentialsProperties(context, action);
- if (credPropertiesMap != null) {
- for (String key : credPropertiesMap.keySet()) {
- CredentialsProperties prop = credPropertiesMap.get(key);
- if (prop != null) {
- LOG.debug("Credential Properties set for action : " + action.getId());
- for (String property : prop.getProperties().keySet()) {
- actionConf.set(property, prop.getProperties().get(property));
- LOG.debug("property : '" + property + "', value : '" + prop.getProperties().get(property) + "'");
+ if (!"true".equals(actionConf.get(OOZIE_CREDENTIALS_SKIP))) {
+ XConfiguration wfJobConf = null;
+ try {
+ wfJobConf = new XConfiguration(new StringReader(context.getWorkflow().getConf()));
+ } catch (IOException ioe) {
+ throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "It should never happen",
+ ioe.getMessage());
+ }
+ if ("false".equals(actionConf.get(OOZIE_CREDENTIALS_SKIP)) ||
+ !wfJobConf.getBoolean(OOZIE_CREDENTIALS_SKIP, ConfigurationService.getBoolean(OOZIE_CREDENTIALS_SKIP))) {
+ credPropertiesMap = getActionCredentialsProperties(context, action);
+ if (credPropertiesMap != null) {
+ for (String key : credPropertiesMap.keySet()) {
+ CredentialsProperties prop = credPropertiesMap.get(key);
+ if (prop != null) {
+ LOG.debug("Credential Properties set for action : " + action.getId());
+ for (String property : prop.getProperties().keySet()) {
+ actionConf.set(property, prop.getProperties().get(property));
+ LOG.debug("property : '" + property + "', value : '" + prop.getProperties().get(property)
+ + "'");
+ }
+ }
}
+ } else {
+ LOG.warn("No credential properties found for action : " + action.getId() + ", cred : " + action.getCred());
}
+ } else {
+ LOG.info("Skipping credentials (" + OOZIE_CREDENTIALS_SKIP + "=true)");
}
+ } else {
+ LOG.info("Skipping credentials (" + OOZIE_CREDENTIALS_SKIP + "=true)");
}
- else {
- LOG.warn("No credential properties found for action : " + action.getId() + ", cred : " + action.getCred());
- }
- }
- else {
+ } else {
LOG.warn("context or action is null");
}
return credPropertiesMap;
http://git-wip-us.apache.org/repos/asf/oozie/blob/1502633e/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index 8a0bc3b..4b9a0bc 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -1932,6 +1932,14 @@
A list of credential class mapping for CredentialsProvider
</description>
</property>
+ <property>
+ <name>oozie.credentials.skip</name>
+ <value>false</value>
+ <description>
+ This determines if Oozie should skip getting credentials from the credential providers. This can be overwritten at a
+ job-level or action-level.
+ </description>
+ </property>
<property>
<name>oozie.actions.main.classnames</name>
http://git-wip-us.apache.org/repos/asf/oozie/blob/1502633e/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 8b867bb..9be2c51 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -29,8 +29,10 @@ import java.io.Writer;
import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
+import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
@@ -938,6 +940,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
HashMap<String, CredentialsProperties> credProperties = ae.setCredentialPropertyToActionConf(context,
action, actionConf);
+ assertNotNull(credProperties);
CredentialsProperties prop = credProperties.get("abcname");
assertEquals("value1", prop.getProperties().get("property1"));
assertEquals("value2", prop.getProperties().get("property2"));
@@ -969,7 +972,123 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
assertNotNull(tk);
}
+ public void testCredentialsSkip() throws Exception {
+ // Try setting oozie.credentials.skip at different levels, and verifying the correct behavior
+ // oozie-site: false -- job-level: null -- action-level: null
+ _testCredentialsSkip(false, null, null, true);
+
+ // oozie-site: false -- job-level: null -- action-level: false
+ _testCredentialsSkip(false, null, "false", true);
+
+ // oozie-site: false -- job-level: null -- action-level: true
+ _testCredentialsSkip(false, null, "true", false);
+
+ // oozie-site: false -- job-level: false -- action-level: null
+ _testCredentialsSkip(false, "false", null, true);
+
+ // oozie-site: false -- job-level: false -- action-level: false
+ _testCredentialsSkip(false, "false", "false", true);
+
+ // oozie-site: false -- job-level: false -- action-level: true
+ _testCredentialsSkip(false, "false", "true", false);
+
+ // oozie-site: false -- job-level: true -- action-level: null
+ _testCredentialsSkip(false, "true", null, false);
+
+ // oozie-site: false -- job-level: true -- action-level: false
+ _testCredentialsSkip(false, "true", "false", true);
+
+ // oozie-site: false -- job-level: true -- action-level: true
+ _testCredentialsSkip(false, "true", "true", false);
+
+ // oozie-site: true -- job-level: null -- action-level: null
+ _testCredentialsSkip(true, null, null, false);
+
+ // oozie-site: true -- job-level: null -- action-level: false
+ _testCredentialsSkip(true, null, "false", true);
+
+ // oozie-site: true -- job-level: null -- action-level: true
+ _testCredentialsSkip(true, null, "true", false);
+
+ // oozie-site: true -- job-level: false -- action-level: null
+ _testCredentialsSkip(true, "false", null, true);
+
+ // oozie-site: true -- job-level: false -- action-level: false
+ _testCredentialsSkip(true, "false", "false", true);
+
+ // oozie-site: true -- job-level: false -- action-level: true
+ _testCredentialsSkip(true, "false", "true", false);
+
+ // oozie-site: true -- job-level: true -- action-level: null
+ _testCredentialsSkip(true, "true", null, false);
+
+ // oozie-site: true -- job-level: true -- action-level: false
+ _testCredentialsSkip(true, "true", "false", true);
+
+ // oozie-site: true -- job-level: true -- action-level: true
+ _testCredentialsSkip(true, "true", "true", false);
+ }
+
+ private void _testCredentialsSkip(boolean skipSite, String skipJob, String skipAction, boolean expectingTokens)
+ throws Exception {
+ String actionLevelSkipConf = (skipAction == null) ? "" :
+ "<property><name>oozie.credentials.skip</name><value>" + skipAction + "</value></property>";
+ String actionxml = "<pig>" + "<job-tracker>${jobTracker}</job-tracker>" + "<name-node>${nameNode}</name-node>"
+ + "<prepare>" + "<delete path='outputdir' />" + "</prepare>" + "<configuration>" + "<property>"
+ + "<name>mapred.compress.map.output</name>" + "<value>true</value>" + "</property>" + "<property>"
+ + "<name>mapred.job.queue.name</name>" + "<value>${queueName}</value>" + "</property>" + actionLevelSkipConf
+ + "</configuration>" + "<script>org/apache/oozie/examples/pig/id.pig</script>"
+ + "<param>INPUT=${inputDir}</param>" + "<param>OUTPUT=${outputDir}/pig-output</param>" + "</pig>";
+ String workflowXml = "<workflow-app xmlns='uri:oozie:workflow:0.2.5' name='pig-wf'>" + "<credentials>"
+ + "<credential name='abcname' type='abc'>" + "<property>" + "<name>property1</name>"
+ + "<value>value1</value>" + "</property>" + "<property>" + "<name>property2</name>"
+ + "<value>value2</value>" + "</property>" + "<property>" + "<name>${property3}</name>"
+ + "<value>${value3}</value>" + "</property>" + "</credential>" + "</credentials>"
+ + "<start to='pig1' />" + "<action name='pig1' cred='abcname'>" + actionxml
+ + "<ok to='end' />" + "<error to='fail' />" + "</action>" + "<kill name='fail'>"
+ + "<message>Pig failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>" + "</kill>"
+ + "<end name='end' />" + "</workflow-app>";
+
+ JavaActionExecutor ae = new JavaActionExecutor();
+ WorkflowJobBean wfBean = addRecordToWfJobTable("test1", workflowXml,
+ (skipJob == null) ? null : Collections.singletonMap("oozie.credentials.skip", skipJob));
+ WorkflowActionBean action = (WorkflowActionBean) wfBean.getActions().get(0);
+ action.setType(ae.getType());
+ action.setCred("abcname");
+ action.setConf(actionxml);
+ Context context = new Context(wfBean, action);
+
+ Element actionXmlconf = XmlUtils.parseXml(action.getConf());
+ // action job configuration
+ Configuration actionConf = ae.createBaseHadoopConf(context, actionXmlconf);
+ actionConf = ae.setupActionConf(actionConf, context, actionXmlconf, new Path("/tmp/foo"));
+
+ // Define 'abc' token type in oozie-site
+ ConfigurationService.set("oozie.credentials.credentialclasses", "abc=org.apache.oozie.action.hadoop.InsertTestToken");
+ ConfigurationService.setBoolean("oozie.credentials.skip", skipSite);
+
+ // Setting the credential properties in launcher conf
+ HashMap<String, CredentialsProperties> credProperties = ae.setCredentialPropertyToActionConf(context,
+ action, actionConf);
+
+ // Try to load the token without it being defined in oozie-site; should get an exception
+ JobConf credentialsConf = new JobConf();
+ Configuration launcherConf = ae.createBaseHadoopConf(context, actionXmlconf);
+ XConfiguration.copy(launcherConf, credentialsConf);
+ ae.setCredentialTokens(credentialsConf, context, action, credProperties);
+ Token<? extends TokenIdentifier> tk = credentialsConf.getCredentials().getToken(new Text("ABC Token"));
+ if (expectingTokens) {
+ assertNotNull(tk);
+ } else {
+ assertNull(tk);
+ }
+ }
+
private WorkflowJobBean addRecordToWfJobTable(String wfId, String wfxml) throws Exception {
+ return addRecordToWfJobTable(wfId, wfxml, null);
+ }
+
+ private WorkflowJobBean addRecordToWfJobTable(String wfId, String wfxml, Map<String, String> otherProps) throws Exception {
WorkflowApp app = new LiteWorkflowApp("testApp", wfxml,
new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "start")).
addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
@@ -980,6 +1099,11 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
conf.set(OozieClient.USER_NAME, getTestUser());
conf.set("property3", "prop3");
conf.set("value3", "val3");
+ if (otherProps != null) {
+ for (Map.Entry<String, String> ent : otherProps.entrySet()) {
+ conf.set(ent.getKey(), ent.getValue());
+ }
+ }
WorkflowJobBean wfBean = createWorkflow(app, conf, "auth");
wfBean.setId(wfId);
http://git-wip-us.apache.org/repos/asf/oozie/blob/1502633e/docs/src/site/twiki/DG_ActionAuthentication.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_ActionAuthentication.twiki b/docs/src/site/twiki/DG_ActionAuthentication.twiki
index 357a5f9..e980e02 100644
--- a/docs/src/site/twiki/DG_ActionAuthentication.twiki
+++ b/docs/src/site/twiki/DG_ActionAuthentication.twiki
@@ -88,6 +88,19 @@ so that Oozie will include these credentials with the action. You can include m
a comma-separated list of =credential= names. And finally, the HCatCredentials required two properties (the metastore URI and
principal), which we also specified.
+Adding the =credentials= section to a workflow and referencing it in an action will make Oozie always try to obtain that delegation
+token. Ordinarily, this would mean that you cannot re-use this workflow in a non-secure cluster without editing it because trying
+to obtain the delegation token will likely fail. However, you can tell Oozie to ignore the =credentials= for a workflow by setting
+the job-level property =oozie.credentials.skip= to =true=; this will allow you to use the same workflow.xml in a secure and
+non-secure cluster by simply changing the job-level property at runtime. If omitted or set to =false=, Oozie will handle
+the =credentials= section normally. In addition, you can also set this property at the action-level or server-level to skip getting
+credentials for just that action or for all workflows, respectively. The order of priority is this:
+
+ 1. =oozie.credentials.skip= in the =configuration= section of an action, if set
+ 1. =oozie.credentials.skip= in the job.properties for a workflow, if set
+ 1. =oozie.credentials.skip= in oozie-site.xml for all workflows, if set
+ 1. (don't skip)
+
---++ Built-in Credentials Implementations
Oozie currently comes with the following Credentials implementations:
http://git-wip-us.apache.org/repos/asf/oozie/blob/1502633e/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 0bd450e..bf84976 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2356 Add a way to enable/disable credentials in a workflow (rkanter)
OOZIE-2355 Hive2 Action doesn't pass along oozie configs to jobconf (rkanter)
OOZIE-2318 Provide better solution for specifying SSL truststore to Oozie Client (rkanter)
OOZIE-2344 Enabling 'oozie.action.jobinfo.enable' doesn't inject the job information into the map/reduce job's configuration. (akshayrai09 via rkanter)