You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by an...@apache.org on 2018/03/26 12:50:21 UTC
oozie git commit: OOZIE-3056 Implement new mechanism to specify
ShareLibs for workflow actions (gezapeti via andras.piros)
Repository: oozie
Updated Branches:
refs/heads/master 50e6c35b4 -> 07deb2b80
OOZIE-3056 Implement new mechanism to specify ShareLibs for workflow actions (gezapeti via andras.piros)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/07deb2b8
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/07deb2b8
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/07deb2b8
Branch: refs/heads/master
Commit: 07deb2b80bf707f9b7841c4c4ca6e677f6c9367f
Parents: 50e6c35
Author: Andras Piros <an...@cloudera.com>
Authored: Mon Mar 26 14:46:39 2018 +0200
Committer: Andras Piros <an...@cloudera.com>
Committed: Mon Mar 26 14:46:39 2018 +0200
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 54 ++------
.../oozie/action/hadoop/SharelibResolver.java | 96 ++++++++++++++
.../action/hadoop/TestJavaActionExecutor.java | 72 +++--------
.../action/hadoop/TestSharelibConfigs.java | 126 +++++++++++++++++++
.../action/hadoop/TestSharelibResolver.java | 72 +++++++++++
.../oozie/action/hadoop/TestWorkflowHelper.java | 103 +++++++++++++++
release-log.txt | 1 +
7 files changed, 431 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/07deb2b8/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 122dfd0..b9c89b2 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -55,7 +55,6 @@ import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.TaskLog;
-import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.AccessControlException;
@@ -164,6 +163,7 @@ public class JavaActionExecutor extends ActionExecutor {
private static final String HADOOP_JOB_NAME = "mapred.job.name";
private static final Set<String> DISALLOWED_PROPERTIES = new HashSet<String>();
private static final String OOZIE_ACTION_NAME = "oozie.action.name";
+ private final static String ACTION_SHARELIB_FOR = "oozie.action.sharelib.for.";
private static int maxActionOutputLen;
private static int maxExternalStatsSize;
@@ -1831,54 +1831,26 @@ public class JavaActionExecutor extends ActionExecutor {
/**
* Return the sharelib names for the action.
- * <p>
- * If <code>NULL</code> or empty, it means that the action does not use the action
- * sharelib.
- * <p>
- * If a non-empty string, i.e. <code>foo</code>, it means the action uses the
- * action sharelib sub-directory <code>foo</code> and all JARs in the sharelib
- * <code>foo</code> directory will be in the action classpath. Multiple sharelib
- * sub-directories can be specified as a comma separated list.
- * <p>
- * The resolution is done using the following precedence order:
- * <ul>
- * <li><b>action.sharelib.for.#ACTIONTYPE#</b> in the action configuration</li>
- * <li><b>action.sharelib.for.#ACTIONTYPE#</b> in the job configuration</li>
- * <li><b>action.sharelib.for.#ACTIONTYPE#</b> in the oozie configuration</li>
- * <li>Action Executor <code>getDefaultShareLibName()</code> method</li>
- * </ul>
- *
+ * See {@link SharelibResolver} for details.
*
* @param context executor context.
- * @param actionXml
+ * @param actionXml the action xml.
* @param conf action configuration.
* @return the action sharelib names.
*/
- protected String[] getShareLibNames(Context context, Element actionXml, Configuration conf) {
- String[] names = conf.getStrings(ACTION_SHARELIB_FOR + getType());
- if (names == null || names.length == 0) {
- try {
- XConfiguration jobConf = getWorkflowConf(context);
- names = jobConf.getStrings(ACTION_SHARELIB_FOR + getType());
- if (names == null || names.length == 0) {
- names = Services.get().getConf().getStrings(ACTION_SHARELIB_FOR + getType());
- if (names == null || names.length == 0) {
- String name = getDefaultShareLibName(actionXml);
- if (name != null) {
- names = new String[] { name };
- }
- }
- }
- }
- catch (IOException ex) {
- throw new RuntimeException("It cannot happen, " + ex.toString(), ex);
- }
+ protected String[] getShareLibNames(final Context context, final Element actionXml, Configuration conf) {
+ final String sharelibFromConfigurationProperty = ACTION_SHARELIB_FOR + getType();
+ try {
+ final String defaultShareLibName = getDefaultShareLibName(actionXml);
+ final Configuration oozieServerConfiguration = Services.get().get(ConfigurationService.class).getConf();
+ final XConfiguration workflowConfiguration = getWorkflowConf(context);
+ return new SharelibResolver(sharelibFromConfigurationProperty, conf, workflowConfiguration,
+ oozieServerConfiguration, defaultShareLibName).resolve();
+ } catch (IOException ex) {
+ throw new RuntimeException("Can't get workflow configuration: " + ex.toString(), ex);
}
- return names;
}
- private final static String ACTION_SHARELIB_FOR = "oozie.action.sharelib.for.";
-
/**
* Returns the default sharelib name for the action if any.
http://git-wip-us.apache.org/repos/asf/oozie/blob/07deb2b8/core/src/main/java/org/apache/oozie/action/hadoop/SharelibResolver.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SharelibResolver.java b/core/src/main/java/org/apache/oozie/action/hadoop/SharelibResolver.java
new file mode 100644
index 0000000..18c50cb
--- /dev/null
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SharelibResolver.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.util.XConfiguration;
+
+import java.io.IOException;
+import java.util.List;
+
+class SharelibResolver {
+ private final String[] defaultValue;
+ private final List<SharelibNameProvider> configProviders;
+
+
+ /**
+ * Return the sharelib names for the action based on the given configurations.
+ * <p>
+ * If <code>NULL</code> or empty, it means that the action does not use the action
+ * sharelib.
+ * <p>
+ * If a non-empty string, i.e. <code>foo</code>, it means the action uses the
+ * action sharelib sub-directory <code>foo</code> and all JARs in the sharelib
+ * <code>foo</code> directory will be in the action classpath. Multiple sharelib
+ * sub-directories can be specified as a comma separated list.
+ * <p>
+ * The resolution is done using the following precedence order:
+ * <ul>
+ * <li><b><sharelib></b> tag in the action's launcher configuration</li>
+ * <li><b>oozie.action.sharelib.for.#ACTIONTYPE#</b> in the action configuration</li>
+ * <li><b><sharelib></b> tag in the workflow's launcher configuration</li>
+ * <li><b>oozie.action.sharelib.for.#ACTIONTYPE#</b> in the workflow configuration</li>
+ * <li><b>oozie.action.sharelib.for.#ACTIONTYPE#</b> in the oozie configuration</li>
+ * <li>Action Executor <code>getDefaultShareLibName()</code> method</li>
+ * </ul>
+ *
+ * @param sharelibPropertyName the property for the current sharelib. E.g. oozie.action.sharelib.for.java
+ * @param actionConf the configuration of the current action
+ * @param workflowConf the global configuration for the workflow
+ * @param oozieServerConfiguration the Oozie server's configuration
+ * @param defaultValue the default value to use if there is no sharelib definition in the configs
+ */
+ SharelibResolver(final String sharelibPropertyName, final Configuration actionConf,
+ final XConfiguration workflowConf, final Configuration oozieServerConfiguration,
+ final String defaultValue) {
+ if (defaultValue == null) {
+ this.defaultValue = new String[0];
+ } else {
+ this.defaultValue = new String[] { defaultValue };
+ }
+ configProviders = Lists.newArrayList(
+ () -> actionConf.getStrings(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY),
+ () -> actionConf.getStrings(sharelibPropertyName),
+ () -> workflowConf.getStrings(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY),
+ () -> workflowConf.getStrings(sharelibPropertyName),
+ () -> oozieServerConfiguration.getStrings(sharelibPropertyName)
+ );
+ }
+
+ /**
+ * Return the sharelib names for the action.
+ * @return the sharelib names
+ */
+ public String[] resolve() {
+ for (SharelibNameProvider cp : configProviders) {
+ if (isValidSharelibProperty(cp.getSharelibNames())) {
+ return cp.getSharelibNames();
+ }
+ }
+ return defaultValue;
+ }
+
+ private boolean isValidSharelibProperty(String[] value){
+ return value != null && value.length > 0;
+ }
+}
+
+interface SharelibNameProvider {
+ String[] getSharelibNames();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/07deb2b8/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index b55a3cd..2bd6a3f 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -89,12 +89,15 @@ import org.jdom.Element;
import org.junit.Assert;
import org.junit.Test;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
public class TestJavaActionExecutor extends ActionExecutorTestCase {
private static final String YARN_RESOURCEMANAGER_ADDRESS = "yarn.resourcemanager.address";
private static final String MAPRED_CHILD_JAVA_OPTS = "mapred.child.java.opts";
private static final String MAPREDUCE_MAP_JAVA_OPTS = "mapreduce.map.java.opts";
-
+ private TestWorkflowHelper helper;
@Override
protected void beforeSetUp() throws Exception {
super.beforeSetUp();
@@ -102,6 +105,12 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
}
@Override
+ public void setUp() throws Exception {
+ super.setUp();
+ helper = new TestWorkflowHelper(getJobTrackerUri(), getNameNodeUri(), getTestCaseDir());
+
+ }
+ @Override
protected void setSystemProps() throws Exception {
super.setSystemProps();
@@ -2247,7 +2256,10 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
public void testDefaultConfigurationInActionConf() throws Exception {
JavaActionExecutor ae = new JavaActionExecutor();
- String xmlStr = getJavaActionXml(true);
+ final String dummyConfiguration = "<configuration>" +
+ "<property><name>action.foo</name><value>AA</value></property>" +
+ "</configuration>";
+ String xmlStr = helper.getJavaActionXml(dummyConfiguration);
Element actionXml = XmlUtils.parseXml(xmlStr);
Context context = createContext(xmlStr, getTestGroup());
Configuration conf = new Configuration(true);
@@ -2261,43 +2273,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
conf.get("mapreduce.map.maxattempts"));
}
- private String getJavaActionXml(boolean addConfig) {
- String config = addConfig ? "<configuration>" +
- "<property><name>action.foo</name><value>AA</value></property>" +
- "</configuration>" : "";
- return "<java>" +
- "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
- "<name-node>" + getNameNodeUri() + "</name-node>" +
- config +
- "<main-class>MAIN-CLASS</main-class>" +
- "</java>";
- }
-
- private String createTestWorkflowXml(final String globalXml, final String actionXml) throws IOException {
- String workflowUri = getTestCaseFileUri("workflow.xml");
- String appXml = "<workflow-app xmlns=\"uri:oozie:workflow:1.0\" name=\"workflow\">" +
- globalXml +
- "<start to=\"java\"/>" +
- "<action name=\"java\">" +
- actionXml +
- " <ok to=\"end\"/>" +
- " <error to=\"fail\"/>" +
- "</action>" +
- "<kill name=\"fail\">" +
- " <message>Sub workflow failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>" +
- "</kill>" +
- "<end name=\"end\"/>" +
- "</workflow-app>";
-
- final File f = new File(URI.create(workflowUri));
- final ByteArrayInputStream inputStream = new ByteArrayInputStream(appXml.getBytes("UTF-8"));
- IOUtils.copyStream(inputStream, new FileOutputStream(f));
- return workflowUri;
- }
-
public void testGlobalConfigurationWithActionDefaults() throws Exception {
try {
- String workflowUri = createTestWorkflowXml(getWorkflowGlobalXml(), getJavaActionXml(false));
+ String workflowUri = helper.createTestWorkflowXml(getWorkflowGlobalXml(), helper.getJavaActionXml(""));
LocalOozie.start();
final OozieClient wfClient = LocalOozie.getClient();
Properties conf = wfClient.createConfiguration();
@@ -2310,11 +2288,11 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
waitFor(20 * 1000, new Predicate() {
@Override
public boolean evaluate() throws Exception {
- WorkflowAction javaAction = getJavaAction(wfClient.getJobInfo(jobId));
+ WorkflowAction javaAction = helper.getJavaAction(wfClient.getJobInfo(jobId));
return javaAction != null && !javaAction.getStatus().equals("PREP");
}
});
- final WorkflowAction workflowAction = getJavaAction(workflow);
+ final WorkflowAction workflowAction = helper.getJavaAction(workflow);
Element eConf = XmlUtils.parseXml(workflowAction.getConf());
Element element = eConf.getChild("configuration", eConf.getNamespace());
Configuration actionConf = new XConfiguration(new StringReader(XmlUtils.prettyPrint(element).toString()));
@@ -2332,7 +2310,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
"<resource-manager>RM</resource-manager>"+
"</global>";
- final String workflowUri = createTestWorkflowXml(global, getJavaActionXml(false));
+ final String workflowUri = helper.createTestWorkflowXml(global, helper.getJavaActionXml(""));
LocalOozie.start();
final OozieClient wfClient = LocalOozie.getClient();
final Properties conf = wfClient.createConfiguration();
@@ -2345,11 +2323,11 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
waitFor(20 * 1000, new Predicate() {
@Override
public boolean evaluate() throws Exception {
- WorkflowAction javaAction = getJavaAction(wfClient.getJobInfo(jobId));
+ WorkflowAction javaAction = helper.getJavaAction(wfClient.getJobInfo(jobId));
return javaAction != null && !javaAction.getStatus().equals("PREP");
}
});
- final WorkflowAction workflowAction = getJavaAction(workflow);
+ final WorkflowAction workflowAction = helper.getJavaAction(workflow);
final String actualConfig = workflowAction.getConf();
final String actualJobTrackerURI = XmlUtils.parseXml(actualConfig).getChildTextNormalize("job-tracker", null);
assertEquals(getJobTrackerUri(), actualJobTrackerURI);
@@ -2359,16 +2337,6 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
}
}
- private WorkflowAction getJavaAction(WorkflowJob workflowJob){
- List<WorkflowAction> actions = workflowJob.getActions();
- for(WorkflowAction wa : actions){
- if(wa.getType().equals("java")){
- return wa;
- }
- }
- return null;
- }
-
public void testSetRootLoggerLevel() throws Exception {
String oozieActionRootLogger = "oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL;
String oozieActionHiveRootLogger = "oozie.action.hive" + LauncherAMUtils.ROOT_LOGGER_LEVEL;
http://git-wip-us.apache.org/repos/asf/oozie/blob/07deb2b8/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibConfigs.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibConfigs.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibConfigs.java
new file mode 100644
index 0000000..bda5ea7
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibConfigs.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.client.WorkflowJob;
+import org.apache.oozie.local.LocalOozie;
+import org.apache.oozie.test.MiniOozieTestCase;
+import org.apache.oozie.util.IOUtils;
+import org.apache.oozie.util.XConfiguration;
+import org.apache.oozie.util.XmlUtils;
+import org.jdom.Element;
+import org.junit.Before;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.StringReader;
+import java.net.URI;
+import java.util.List;
+import java.util.Properties;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+public class TestSharelibConfigs extends MiniOozieTestCase {
+
+ private TestWorkflowHelper helper;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ helper = new TestWorkflowHelper(getJobTrackerUri(), getNameNodeUri(), getTestCaseDir());
+ }
+
+ public void testActionSharelibConfigPropagation() throws Exception {
+ final String globalConfigForJavaSharelib = "<global><configuration><property>" +
+ "<name>oozie.action.sharelib.for.java</name><value>globalJavaConfigSharelib</value>" +
+ "</property></configuration></global>";
+ final String globalLauncherWithSharelib = "<global><launcher>" +
+ "<sharelib>globalLauncherSharelib</sharelib>" +
+ "</launcher></global>";
+ final String globalLauncherAndConfigWithSharelib = "<global><launcher>" +
+ "<sharelib>globalLauncherSharelib</sharelib>" +
+ "</launcher><configuration><property>" +
+ "<name>oozie.action.sharelib.for.java</name><value>globalJavaConfigSharelib</value>" +
+ "</property></configuration></global>";
+ final String localConfigForJavaSharelib = "<configuration><property>" +
+ "<name>oozie.action.sharelib.for.java</name><value>localJavaConfigSharelib</value>" +
+ "</property></configuration>";
+ final String localLauncherWithSharelib = "<launcher>" +
+ "<sharelib>localLauncherSharelib</sharelib>" +
+ "</launcher>";
+
+ launchJavaActionAndValidateSharelibValues("", "",
+ null, null);
+ launchJavaActionAndValidateSharelibValues(globalConfigForJavaSharelib, "",
+ "globalJavaConfigSharelib", null);
+ launchJavaActionAndValidateSharelibValues(globalLauncherWithSharelib, "",
+ null, "globalLauncherSharelib");
+ launchJavaActionAndValidateSharelibValues(globalLauncherAndConfigWithSharelib, localConfigForJavaSharelib,
+ "localJavaConfigSharelib", "globalLauncherSharelib");
+ launchJavaActionAndValidateSharelibValues(globalLauncherAndConfigWithSharelib, localLauncherWithSharelib,
+ "globalJavaConfigSharelib", "localLauncherSharelib");
+ launchJavaActionAndValidateSharelibValues(globalLauncherAndConfigWithSharelib,
+ localLauncherWithSharelib + localConfigForJavaSharelib,
+ "localJavaConfigSharelib", "localLauncherSharelib");
+
+ }
+
+ private void launchJavaActionAndValidateSharelibValues(String globalConfig,
+ String localConfig,
+ String oozieSharelibForJavaPropertyValue,
+ String oozieSharelibPropertyValue)
+ throws Exception {
+ final String workflowUri = helper.createTestWorkflowXml(globalConfig,
+ helper.getJavaActionXml(localConfig));
+ final OozieClient wfClient = LocalOozie.getClient();
+ final Properties conf = wfClient.createConfiguration();
+ conf.setProperty(OozieClient.APP_PATH, workflowUri);
+ conf.setProperty(OozieClient.USER_NAME, getTestUser());
+ conf.setProperty("appName", "var-app-name");
+ conf.setProperty(OozieClient.USE_SYSTEM_LIBPATH, "true");
+ final String jobId = wfClient.submit(conf);
+ wfClient.start(jobId);
+ WorkflowJob workflow = wfClient.getJobInfo(jobId);
+ waitFor(20 * 1000, new Predicate() {
+ @Override
+ public boolean evaluate() throws Exception {
+ WorkflowAction javaAction = helper.getJavaAction(wfClient.getJobInfo(jobId));
+ return javaAction != null && !javaAction.getStatus().equals("PREP");
+ }
+ });
+ final XConfiguration actionConf = getJavaActionConfiguration(workflow);
+ assertThat("Configuration priorities are incorrect! Global/local configs are not overwriting each other.",
+ actionConf.get(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY), is(oozieSharelibPropertyValue));
+ assertThat("Configuration priorities are incorrect! Global/local configs are not overwriting each other.",
+ actionConf.get("oozie.action.sharelib.for.java"), is(oozieSharelibForJavaPropertyValue));
+ }
+
+ private XConfiguration getJavaActionConfiguration(WorkflowJob workflow) throws Exception {
+ final WorkflowAction workflowAction = helper.getJavaAction(workflow);
+ final Element element = XmlUtils.parseXml(workflowAction.getConf());
+ final String configuration = XmlUtils.prettyPrint(element.getChild("configuration",
+ element.getNamespace())).toString();
+ return new XConfiguration(new StringReader(configuration));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/07deb2b8/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibResolver.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibResolver.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibResolver.java
new file mode 100644
index 0000000..985a250
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestSharelibResolver.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.util.XConfiguration;
+import org.junit.Test;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+public class TestSharelibResolver {
+ public final String defaultValue = "default";
+ private final String serverConfigSharelib = "serverConfigSharelib";
+ private final String globalConfigSharelib = "globalConfigSharelib";
+ private final String globalLauncherSharelib = "globalLauncherSharelib";
+ private final String localConfigSharelib = "localConfigSharelib";
+ private final String localLauncherSharelib = "localLauncherSharelib";
+ private final String sharelibProperty = "sharelibProperty";
+
+ /**
+ * Add sharelib values in reverse priority order and check that the last added value is the one that gets resolved.
+ */
+ @Test
+ public void testResolvingOrder() {
+ Configuration oozieServerConfiguration = new Configuration(false);
+ XConfiguration workflowConf = new XConfiguration();
+ Configuration actionConf = new Configuration(false);
+
+ SharelibResolver resolver =
+ new SharelibResolver(sharelibProperty, actionConf, workflowConf, oozieServerConfiguration, defaultValue);
+
+ assertThat("Without setting anything we should've got the default value.",
+ resolver.resolve(), is(new String[]{defaultValue}));
+
+ oozieServerConfiguration.set(sharelibProperty, serverConfigSharelib);
+ assertThat("Server-level sharelib configuration is not processed",
+ resolver.resolve(), is(new String[]{serverConfigSharelib}));
+
+ workflowConf.set(sharelibProperty, globalConfigSharelib);
+ assertThat("Global workflow-level sharelib configuration is not processed",
+ resolver.resolve(), is(new String[]{globalConfigSharelib}));
+
+ workflowConf.set(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY, globalLauncherSharelib);
+ assertThat("Global workflow-level sharelib configuration is not processed",
+ resolver.resolve(), is(new String[]{globalLauncherSharelib}));
+
+ actionConf.set(sharelibProperty, localConfigSharelib);
+ assertThat("Local action-level sharelib configuration is not processed",
+ resolver.resolve(), is(new String[]{localConfigSharelib}));
+
+ actionConf.set(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY, localLauncherSharelib);
+ assertThat("Local action-level sharelib configuration is not processed",
+ resolver.resolve(), is(new String[]{localLauncherSharelib}));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/07deb2b8/core/src/test/java/org/apache/oozie/action/hadoop/TestWorkflowHelper.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestWorkflowHelper.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestWorkflowHelper.java
new file mode 100644
index 0000000..d6bebe4
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestWorkflowHelper.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.client.WorkflowJob;
+import org.apache.oozie.test.MiniOozieTestCase;
+import org.apache.oozie.test.XTestCase;
+import org.apache.oozie.util.IOUtils;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+
+public class TestWorkflowHelper {
+
+ private final String jobTrackerUri;
+ private final String nameNodeUri;
+ private final String testCaseDir;
+
+
+ public TestWorkflowHelper(String jobTrackerUri, String nameNodeUri, String testCaseDir) {
+ this.jobTrackerUri = jobTrackerUri;
+ this.nameNodeUri = nameNodeUri;
+ this.testCaseDir = testCaseDir;
+ }
+
+ protected String getJavaActionXml(String configToAdd) {
+ return "<java>" +
+ "<job-tracker>" + jobTrackerUri + "</job-tracker>" +
+ "<name-node>" + nameNodeUri + "</name-node>" +
+ configToAdd +
+ "<main-class>MAIN-CLASS</main-class>" +
+ "</java>";
+ }
+
+ protected String createTestWorkflowXml(final String globalXml, final String actionXml) throws IOException {
+ String workflowUri = getTestCaseFileUri("workflow.xml");
+ String appXml = "<workflow-app xmlns=\"uri:oozie:workflow:1.0\" name=\"workflow\">" +
+ globalXml +
+ "<start to=\"java\"/>" +
+ "<action name=\"java\">" +
+ actionXml +
+ " <ok to=\"end\"/>" +
+ " <error to=\"fail\"/>" +
+ "</action>" +
+ "<kill name=\"fail\">" +
+ " <message>Sub workflow failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>" +
+ "</kill>" +
+ "<end name=\"end\"/>" +
+ "</workflow-app>";
+ final File f = new File(URI.create(workflowUri));
+ final ByteArrayInputStream inputStream = new ByteArrayInputStream(appXml.getBytes("UTF-8"));
+ IOUtils.copyStream(inputStream, new FileOutputStream(f));
+ return workflowUri;
+ }
+
+ protected WorkflowAction getJavaAction(WorkflowJob workflowJob){
+ List<WorkflowAction> actions = workflowJob.getActions();
+ for(WorkflowAction wa : actions){
+ if(wa.getType().equals("java")){
+ return wa;
+ }
+ }
+ return null;
+ }
+ /**
+ * Return the URI for a test file. The returned value is the testDir + concatenated URI.
+ *
+ * @return the test working directory path, it is always an absolute path and appends the relative path. The
+ * reason for the manual parsing instead of an actual File.toURI is because Oozie tests use tokens ${}
+ * frequently. Something like URI("c:/temp/${HOUR}").toString() will generate escaped values that will break tests
+ */
+ private String getTestCaseFileUri(String relativeUri) {
+ String uri = new File(testCaseDir).toURI().toString();
+
+ // truncates '/' if the testCaseDir was provided with a fullpath ended with separator
+ if (uri.endsWith("/")){
+ uri = uri.substring(0, uri.length() -1);
+ }
+
+ return uri + "/" + relativeUri;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/07deb2b8/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 76401f5..d126338 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.0.0 release (trunk - unreleased)
+OOZIE-3056 Implement new mechanism to specify ShareLibs for workflow actions (gezapeti via andras.piros)
OOZIE-2600 OYA: Update Documentation (andras.piros via gezapeti)
OOZIE-3189 Update the release script and wiki page to use sha512 instead of md5 (rkanter via gezapeti)
OOZIE-3195 Typo in WebServicesAPI.twiki: Proxy Hive Job Submission (kmarton via andras.piros)