You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by an...@apache.org on 2012/01/06 22:57:23 UTC
svn commit: r1228429 - in /incubator/oozie/branches/3.1/core/src:
main/java/org/apache/oozie/service/WorkflowAppService.java
test/java/org/apache/oozie/action/hadoop/PigTestCase.java
test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
Author: angeloh
Date: Fri Jan 6 21:57:22 2012
New Revision: 1228429
URL: http://svn.apache.org/viewvc?rev=1228429&view=rev
Log:
Closes OOZIE-564 Embedded pig within python fails when run using oozie -- Adding new file
Added:
incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/PigTestCase.java
Modified:
incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
Modified: incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java?rev=1228429&r1=1228428&r2=1228429&view=diff
==============================================================================
--- incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java (original)
+++ incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java Fri Jan 6 21:57:22 2012
@@ -201,6 +201,11 @@ public abstract class WorkflowAppService
if (entry.getKey().startsWith("oozie.")) {
String name = entry.getKey();
String value = entry.getValue();
+ // Append application lib jars of both parent and child in
+ // subworkflow to APP_LIB_PATH_LIST
+ if ((conf.get(name) != null) && name.equals(APP_LIB_PATH_LIST)) {
+ value = value + "," + conf.get(name);
+ }
conf.set(name, value);
}
}
Added: incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/PigTestCase.java
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/PigTestCase.java?rev=1228429&view=auto
==============================================================================
--- incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/PigTestCase.java (added)
+++ incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/PigTestCase.java Fri Jan 6 21:57:22 2012
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.concurrent.Callable;
+
+import org.antlr.runtime.ANTLRReaderStream;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.test.XFsTestCase;
+import org.apache.oozie.util.ClassUtils;
+import org.apache.oozie.util.IOUtils;
+import org.python.util.jython;
+
+import com.google.common.primitives.Booleans;
+
+public abstract class PigTestCase extends XFsTestCase implements Callable<Void> {
+ protected static String pigScript;
+ private static String commonPigScript = "set job.name 'test'\n" +
+ "set debug on\n" +
+ "A = load '$IN' using PigStorage(':');\n" +
+ "B = foreach A generate $0 as id;\n" +
+ "store B into '$OUT' USING PigStorage();";
+
+ public void testPigScript() throws Exception {
+ pigScript = commonPigScript;
+ DoAs.call(getTestUser(), this);
+
+ }
+
+ // testing embedded Pig feature of Pig 0.9
+ public void testEmbeddedPigWithinPython() throws Exception {
+ FileSystem fs = getFileSystem();
+ Path jythonJar = new Path(getFsTestCaseDir(), "jython.jar");
+ InputStream is = new FileInputStream(ClassUtils.findContainingJar(jython.class));
+ OutputStream os = fs.create(jythonJar);
+ IOUtils.copyStream(is, os);
+
+ Path antlrRuntimeJar = new Path(getFsTestCaseDir(), "antlr_runtime.jar");
+ is = new FileInputStream(ClassUtils.findContainingJar(ANTLRReaderStream.class));
+ os = fs.create(antlrRuntimeJar);
+ IOUtils.copyStream(is, os);
+
+ Path guavaJar = new Path(getFsTestCaseDir(), "guava.jar");
+ is = new FileInputStream(ClassUtils.findContainingJar(Booleans.class));
+ os = fs.create(guavaJar);
+ IOUtils.copyStream(is, os);
+
+ DistributedCache.addFileToClassPath(new Path(jythonJar.toUri().getPath()), getFileSystem().getConf());
+ DistributedCache.addFileToClassPath(new Path(antlrRuntimeJar.toUri().getPath()), getFileSystem().getConf());
+ DistributedCache.addFileToClassPath(new Path(guavaJar.toUri().getPath()), getFileSystem().getConf());
+
+ Path inputDir = new Path(getFsTestCaseDir(), "input");
+ Path outputDir = new Path(getFsTestCaseDir(), "output");
+ pigScript = "#!/usr/bin/python"
+ + "\nfrom org.apache.pig.scripting import Pig"
+ + "\nP = Pig.compile(\"\"\"" + "\n" + commonPigScript + "\n"+ "\"\"\")"
+ + "\ninput = " + "'"+inputDir.toUri().getPath()+"'"
+ + "\noutput = " + "'"+outputDir.toUri().getPath()+"'"
+ + "\nQ = P.bind({'IN':input, 'OUT':output})"
+ + "\nQ.runSingle()";
+ DoAs.call(getTestUser(), this);
+ }
+
+}
Modified: incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java?rev=1228429&r1=1228428&r2=1228429&view=diff
==============================================================================
--- incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java (original)
+++ incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java Fri Jan 6 21:57:22 2012
@@ -551,4 +551,89 @@ public class TestLiteWorkflowAppService
}
}
+ public void testCreateprotoConfWithSubWorkflow_Case1_ParentWorkflowContainingLibs() throws Exception {
+ // When parent workflow has an non-empty lib directory,
+ // APP_LIB_PATH_LIST should contain libraries from both parent and
+ // subworkflow (child)
+ Services services = new Services();
+ try {
+ services.init();
+ Reader reader = IOUtils.getResourceAsReader("wf-schema-valid.xml", -1);
+ Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
+ IOUtils.copyCharStream(reader, writer);
+
+ createTestCaseSubDir("lib");
+ writer = new FileWriter(getTestCaseDir() + "/lib/childdependency1.jar");
+ writer.write("bla bla");
+ writer.close();
+ writer = new FileWriter(getTestCaseDir() + "/lib/childdependency2.so");
+ writer.write("bla bla");
+ writer.close();
+ WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
+ Configuration jobConf = new XConfiguration();
+ jobConf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() + File.separator + "workflow.xml");
+ jobConf.set(OozieClient.USER_NAME, getTestUser());
+ jobConf.set(OozieClient.GROUP_NAME, getTestGroup());
+ jobConf.set(WorkflowAppService.APP_LIB_PATH_LIST, "parentdependency1.jar");
+ injectKerberosInfo(jobConf);
+ Configuration protoConf = wps.createProtoActionConf(jobConf, "authToken", true);
+ assertEquals(getTestUser(), protoConf.get(OozieClient.USER_NAME));
+ assertEquals(getTestGroup(), protoConf.get(OozieClient.GROUP_NAME));
+
+ assertEquals(3, protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST).length);
+ String f1 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[0];
+ String f2 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[1];
+ String f3 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[2];
+ String ref1 = "parentdependency1.jar";
+ String ref2 = getTestCaseDir() + "/lib/childdependency1.jar";
+ String ref3 = getTestCaseDir() + "/lib/childdependency2.so";
+ Assert.assertTrue(f1.equals(ref1));
+ Assert.assertTrue(f2.equals(ref2));
+ Assert.assertTrue(f3.equals(ref3));
+ }
+ finally {
+ services.destroy();
+ }
+ }
+
+ public void testCreateprotoConfWithSubWorkflow_Case2_ParentWorkflowWithoutLibs() throws Exception {
+ // When parent workflow has an empty (or missing) lib directory,
+ // APP_LIB_PATH_LIST should contain libraries from only the subworkflow
+ // (child)
+ Services services = new Services();
+ try {
+ services.init();
+ Reader reader = IOUtils.getResourceAsReader("wf-schema-valid.xml", -1);
+ Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
+ IOUtils.copyCharStream(reader, writer);
+
+ createTestCaseSubDir("lib");
+ writer = new FileWriter(getTestCaseDir() + "/lib/childdependency1.jar");
+ writer.write("bla bla");
+ writer.close();
+ writer = new FileWriter(getTestCaseDir() + "/lib/childdependency2.so");
+ writer.write("bla bla");
+ writer.close();
+ WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
+ Configuration jobConf = new XConfiguration();
+ jobConf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() + File.separator + "workflow.xml");
+ jobConf.set(OozieClient.USER_NAME, getTestUser());
+ jobConf.set(OozieClient.GROUP_NAME, getTestGroup());
+ injectKerberosInfo(jobConf);
+ Configuration protoConf = wps.createProtoActionConf(jobConf, "authToken", true);
+ assertEquals(getTestUser(), protoConf.get(OozieClient.USER_NAME));
+ assertEquals(getTestGroup(), protoConf.get(OozieClient.GROUP_NAME));
+
+ assertEquals(2, protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST).length);
+ String f1 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[0];
+ String f2 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[1];
+ String ref1 = getTestCaseDir() + "/lib/childdependency1.jar";
+ String ref2 = getTestCaseDir() + "/lib/childdependency2.so";
+ Assert.assertTrue(f1.equals(ref1));
+ Assert.assertTrue(f2.equals(ref2));
+ }
+ finally {
+ services.destroy();
+ }
+ }
}