You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2013/02/15 07:14:22 UTC
svn commit: r1446455 - in /oozie/trunk:
core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
docs/src/site/twiki/WorkflowFunctionalSpec.twiki release-log.txt
Author: rkanter
Date: Fri Feb 15 06:14:22 2013
New Revision: 1446455
URL: http://svn.apache.org/r1446455
Log:
OOZIE-1226 Workflow lib path not found in classpath for a subworkflow (rkanter)
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
oozie/trunk/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
oozie/trunk/release-log.txt
Modified: oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java?rev=1446455&r1=1446454&r2=1446455&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java (original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java Fri Feb 15 06:14:22 2013
@@ -37,8 +37,11 @@ import java.io.Reader;
import java.io.StringWriter;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedHashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -57,8 +60,13 @@ public abstract class WorkflowAppService
public static final String CONFG_MAX_WF_LENGTH = CONF_PREFIX + "WorkflowDefinitionMaxLength";
+ public static final String OOZIE_SUBWORKFLOW_CLASSPATH_INHERITANCE = "oozie.subworkflow.classpath.inheritance";
+
+ public static final String OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE = "oozie.wf.subworkflow.classpath.inheritance";
+
private Path systemLibPath;
private long maxWFLength;
+ private boolean oozieSubWfCPInheritance;
/**
* Initialize the workflow application service.
@@ -74,6 +82,8 @@ public abstract class WorkflowAppService
}
maxWFLength = conf.getInt(CONFG_MAX_WF_LENGTH, 100000);
+
+ oozieSubWfCPInheritance = conf.getBoolean(OOZIE_SUBWORKFLOW_CLASSPATH_INHERITANCE, false);
}
/**
@@ -195,6 +205,30 @@ public abstract class WorkflowAppService
}
}
+ // Check if a subworkflow should inherit the libs from the parent WF
+ // OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE has priority over OOZIE_SUBWORKFLOW_CLASSPATH_INHERITANCE from oozie-site
+ // If OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE isn't specified, we use OOZIE_SUBWORKFLOW_CLASSPATH_INHERITANCE
+ if (jobConf.getBoolean(OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE, oozieSubWfCPInheritance)) {
+ // Keep any libs from a parent workflow that might already be in APP_LIB_PATH_LIST and also remove duplicates
+ String[] parentFilePaths = jobConf.getStrings(APP_LIB_PATH_LIST);
+ if (parentFilePaths != null && parentFilePaths.length > 0) {
+ String[] filePathsNames = filePaths.toArray(new String[filePaths.size()]);
+ for (int i = 0; i < filePathsNames.length; i++) {
+ Path p = new Path(filePathsNames[i]);
+ filePathsNames[i] = p.getName();
+ }
+ Arrays.sort(filePathsNames);
+ List<String> nonDuplicateParentFilePaths = new ArrayList<String>();
+ for (String parentFilePath : parentFilePaths) {
+ Path p = new Path(parentFilePath);
+ if (Arrays.binarySearch(filePathsNames, p.getName()) < 0) {
+ nonDuplicateParentFilePaths.add(parentFilePath);
+ }
+ }
+ filePaths.addAll(nonDuplicateParentFilePaths);
+ }
+ }
+
conf.setStrings(APP_LIB_PATH_LIST, filePaths.toArray(new String[filePaths.size()]));
//Add all properties start with 'oozie.'
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java?rev=1446455&r1=1446454&r2=1446455&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java Fri Feb 15 06:14:22 2013
@@ -35,10 +35,12 @@ import java.io.FileWriter;
import java.io.Reader;
import java.io.Writer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import junit.framework.Assert;
+import org.apache.hadoop.fs.Path;
import org.apache.oozie.workflow.lite.StartNodeDef;
public class TestLiteWorkflowAppService extends XTestCase {
@@ -452,46 +454,191 @@ public class TestLiteWorkflowAppService
}
}
- public void testCreateprotoConfWithSubWorkflow_CheckSubworkflowLibInClasspath() throws Exception {
- // When subworkflow has non-empty lib directory, APP_LIB_PATH_LIST for
- // subworkflow should maintain its corresponding libraries in path
+ private static String[] parentLibs1 = {"parent1.jar", "parent2.jar"};
+ private static String[] childLibs1 = {"child1.jar", "child2.so"};
+ private static String[] parentLibs2 = {"parent1.jar", "parent2.jar"};
+ private static String[] childLibs2 = {};;
+ private static String[] parentLibs3 = {};;
+ private static String[] childLibs3 = {"child1.jar", "child2.so"};;
+ private static String[] parentLibs4 = {};;
+ private static String[] childLibs4 = {};;
+ private static String[] parentLibs5 = {"parent1.jar", "parent2.jar", "same.jar"};;
+ private static String[] childLibs5 = {"child1.jar", "same.jar", "child2.so"};;
+
+ public void testCreateProtoConfWithSubWorkflowLib1() throws Exception {
+ String inherit = "true";
+ String inheritWF = null;
+
+ String[] expectedLibs1 = {"parent1.jar", "parent2.jar", "child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 1, parentLibs1, childLibs1, expectedLibs1);
+
+ String[] expectedLibs2 = {"parent1.jar", "parent2.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 2, parentLibs2, childLibs2, expectedLibs2);
+
+ String[] expectedLibs3 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 3, parentLibs3, childLibs3, expectedLibs3);
+
+ String[] expectedLibs4 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 4, parentLibs4, childLibs4, expectedLibs4);
+
+ String[] expectedLibs5 = {"parent1.jar", "parent2.jar", "child1.jar", "child2.so", "same.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 5, parentLibs5, childLibs5, expectedLibs5);
+ }
+
+ public void testCreateProtoConfWithSubWorkflowLib2() throws Exception {
+ String inherit = "false";
+ String inheritWF = null;
+
+ String[] expectedLibs1 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 1, parentLibs1, childLibs1, expectedLibs1);
+
+ String[] expectedLibs2 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 2, parentLibs2, childLibs2, expectedLibs2);
+
+ String[] expectedLibs3 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 3, parentLibs3, childLibs3, expectedLibs3);
+
+ String[] expectedLibs4 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 4, parentLibs4, childLibs4, expectedLibs4);
+
+ String[] expectedLibs5 = {"child1.jar", "child2.so", "same.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 5, parentLibs5, childLibs5, expectedLibs5);
+ }
+
+ public void testCreateProtoConfWithSubWorkflowLib3() throws Exception {
+ String inherit = "true";
+ String inheritWF = "true";
+
+ String[] expectedLibs1 = {"parent1.jar", "parent2.jar", "child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 1, parentLibs1, childLibs1, expectedLibs1);
+
+ String[] expectedLibs2 = {"parent1.jar", "parent2.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 2, parentLibs2, childLibs2, expectedLibs2);
+
+ String[] expectedLibs3 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 3, parentLibs3, childLibs3, expectedLibs3);
+
+ String[] expectedLibs4 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 4, parentLibs4, childLibs4, expectedLibs4);
+
+ String[] expectedLibs5 = {"parent1.jar", "parent2.jar", "child1.jar", "child2.so", "same.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 5, parentLibs5, childLibs5, expectedLibs5);
+ }
+
+ public void testCreateProtoConfWithSubWorkflowLib4() throws Exception {
+ String inherit = "false";
+ String inheritWF = "true";
+
+ String[] expectedLibs1 = {"parent1.jar", "parent2.jar", "child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 1, parentLibs1, childLibs1, expectedLibs1);
+
+ String[] expectedLibs2 = {"parent1.jar", "parent2.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 2, parentLibs2, childLibs2, expectedLibs2);
+
+ String[] expectedLibs3 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 3, parentLibs3, childLibs3, expectedLibs3);
+
+ String[] expectedLibs4 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 4, parentLibs4, childLibs4, expectedLibs4);
+
+ String[] expectedLibs5 = {"parent1.jar", "parent2.jar", "child1.jar", "child2.so", "same.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 5, parentLibs5, childLibs5, expectedLibs5);
+ }
+
+ public void testCreateProtoConfWithSubWorkflowLib5() throws Exception {
+ String inherit = "true";
+ String inheritWF = "false";
+
+ String[] expectedLibs1 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 1, parentLibs1, childLibs1, expectedLibs1);
+
+ String[] expectedLibs2 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 2, parentLibs2, childLibs2, expectedLibs2);
+
+ String[] expectedLibs3 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 3, parentLibs3, childLibs3, expectedLibs3);
+
+ String[] expectedLibs4 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 4, parentLibs4, childLibs4, expectedLibs4);
+
+ String[] expectedLibs5 = {"child1.jar", "child2.so", "same.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 5, parentLibs5, childLibs5, expectedLibs5);
+ }
+
+ public void testCreateProtoConfWithSubWorkflowLib6() throws Exception {
+ String inherit = "false";
+ String inheritWF = "false";
+
+ String[] expectedLibs1 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 1, parentLibs1, childLibs1, expectedLibs1);
+
+ String[] expectedLibs2 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 2, parentLibs2, childLibs2, expectedLibs2);
+
+ String[] expectedLibs3 = {"child1.jar", "child2.so"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 3, parentLibs3, childLibs3, expectedLibs3);
+
+ String[] expectedLibs4 = {};
+ checkSubworkflowLibHelper(inherit, inheritWF, 4, parentLibs4, childLibs4, expectedLibs4);
+
+ String[] expectedLibs5 = {"child1.jar", "child2.so", "same.jar"};
+ checkSubworkflowLibHelper(inherit, inheritWF, 5, parentLibs5, childLibs5, expectedLibs5);
+ }
+
+ public void checkSubworkflowLibHelper(String inherit, String inheritWF, int unique, String[] parentLibs, String[] childLibs,
+ String[] expectedLibs) throws Exception {
Services services = new Services();
try {
+ services.getConf().set("oozie.subworkflow.classpath.inheritance", inherit);
services.init();
Reader reader = IOUtils.getResourceAsReader("wf-schema-valid.xml", -1);
- Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
+ String childWFDir = createTestCaseSubDir("child-wf-" + unique);
+ Writer writer = new FileWriter(childWFDir + File.separator + "workflow.xml");
IOUtils.copyCharStream(reader, writer);
- createTestCaseSubDir("lib");
- writer = new FileWriter(getTestCaseDir() + "/lib/childdependency1.jar");
- writer.write("bla bla");
- writer.close();
- writer = new FileWriter(getTestCaseDir() + "/lib/childdependency2.so");
- writer.write("bla bla");
- writer.close();
WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
Configuration jobConf = new XConfiguration();
- jobConf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() + File.separator + "workflow.xml");
+ jobConf.set(OozieClient.APP_PATH, "file://" + childWFDir + File.separator + "workflow.xml");
jobConf.set(OozieClient.USER_NAME, getTestUser());
- jobConf.set(WorkflowAppService.APP_LIB_PATH_LIST, "parentdependency1.jar");
+ if (inheritWF != null) {
+ jobConf.set("oozie.wf.subworkflow.classpath.inheritance", inheritWF);
+ }
+
+ String childLibDir = createTestCaseSubDir("child-wf-" + unique + File.separator + "lib");
+ for (String childLib : childLibs) {
+ writer = new FileWriter(childLibDir + File.separator + childLib);
+ writer.write("bla bla");
+ writer.close();
+ }
+ String parentWFDir = createTestCaseSubDir("parent-wf-" + unique);
+ String parentLibDir = createTestCaseSubDir("parent-wf-" + unique + File.separator + "lib");
+ String[] parentLibsFullPaths = new String[parentLibs.length];
+ for (int i = 0; i < parentLibs.length; i++) {
+ parentLibsFullPaths[i] = parentLibDir + File.separator + parentLibs[i];
+ writer = new FileWriter(parentLibsFullPaths[i]);
+ writer.write("bla bla");
+ writer.close();
+ }
+ // Set the parent libs
+ jobConf.setStrings(WorkflowAppService.APP_LIB_PATH_LIST, parentLibsFullPaths);
Configuration protoConf = wps.createProtoActionConf(jobConf, "authToken", true);
assertEquals(getTestUser(), protoConf.get(OozieClient.USER_NAME));
- assertEquals(2, protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST).length);
- String f1 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[0];
- String f2 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[1];
- String ref1 = "file://" + getTestCaseDir() + "/lib/childdependency1.jar";
- String ref2 = "file://" + getTestCaseDir() + "/lib/childdependency2.so";
- List<String> expected = new ArrayList<String>();
- expected.add(ref1);
- expected.add(ref2);
- List<String> found = new ArrayList<String>();
- found.add(f1);
- found.add(f2);
- Collections.sort(found);
- Collections.sort(expected);
- assertEquals(expected, found);
+ String[] foundLibs = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST);
+ if (expectedLibs.length > 0) {
+ assertEquals(expectedLibs.length, foundLibs.length);
+ for (int i = 0; i < foundLibs.length; i++) {
+ Path p = new Path(foundLibs[i]);
+ foundLibs[i] = p.getName();
+ }
+ Arrays.sort(expectedLibs);
+ Arrays.sort(foundLibs);
+ assertEquals(Arrays.toString(expectedLibs), Arrays.toString(foundLibs));
+ }
+ else {
+ assertEquals(null, foundLibs);
+ }
}
finally {
services.destroy();
Modified: oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
URL: http://svn.apache.org/viewvc/oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki?rev=1446455&r1=1446454&r2=1446455&view=diff
==============================================================================
--- oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki (original)
+++ oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki Fri Feb 15 06:14:22 2013
@@ -1280,6 +1280,12 @@ In the above example, the workflow defin
A configuration parameter =input.dir= is being passed as job property to the child workflow job.
+The subworkflow can inherit the lib jars from the parent workflow by setting =oozie.subworkflow.classpath.inheritance= to true
+in oozie-site.xml or on a per-job basis by setting =oozie.wf.subworkflow.classpath.inheritance= to true in a job.properties file.
+If both are specified, =oozie.wf.subworkflow.classpath.inheritance= has priority. If the subworkflow and the parent have
+conflicting jars, the subworkflow's jar has priority. By default, =oozie.wf.subworkflow.classpath.inheritance= is set to false.
+
+
#JavaAction
---++++ 3.2.7 Java Action
Modified: oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1446455&r1=1446454&r2=1446455&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Fri Feb 15 06:14:22 2013
@@ -13,6 +13,7 @@ OOZIE-944 Implement Workflow Generator U
-- Oozie 3.3.2 (unreleased)
+OOZIE-1226 Workflow lib path not found in classpath for a subworkflow (rkanter)
OOZIE-1184 Demo example job.properties has an unused parameter (udai via rkanter)
OOZIE-1211 oozie does not support duplicated dataset (jaoki via virag)
OOZIE-1187 reduce memory usage of SLA query (invoked by CLI command) to avoid OOM (egashira via virag)