You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by tu...@apache.org on 2012/02/22 20:55:51 UTC
svn commit: r1292481 - in /incubator/oozie/trunk: ./
core/src/test/java/org/apache/oozie/action/hadoop/ sharelib/hive/
sharelib/oozie/ sharelib/pig/ sharelib/sqoop/ sharelib/streaming/
Author: tucu
Date: Wed Feb 22 19:55:51 2012
New Revision: 1292481
URL: http://svn.apache.org/viewvc?rev=1292481&view=rev
Log:
OOZIE-704 Sharelib action JARs for testcases should be collected leveraging Maven dependency information (tucu)
Added:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/SharelibUtils.java
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java
incubator/oozie/trunk/pom.xml
incubator/oozie/trunk/release-log.txt
incubator/oozie/trunk/sharelib/hive/pom.xml
incubator/oozie/trunk/sharelib/oozie/pom.xml
incubator/oozie/trunk/sharelib/pig/pom.xml
incubator/oozie/trunk/sharelib/sqoop/pom.xml
incubator/oozie/trunk/sharelib/streaming/pom.xml
Added: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/SharelibUtils.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/SharelibUtils.java?rev=1292481&view=auto
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/SharelibUtils.java (added)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/SharelibUtils.java Wed Feb 22 19:55:51 2012
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.util.IOUtils;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class SharelibUtils {
+
+ private static String findDir(String name) throws Exception {
+ return findDir(new File("foo").getAbsoluteFile().getParent(), name);
+ }
+
+ private static String findDir(String baseDir, String name) throws Exception {
+ File dir = new File(baseDir, name).getAbsoluteFile();
+ if (!dir.exists()) {
+ File parent = dir.getParentFile().getParentFile();
+ if (parent != null) {
+ return findDir(parent.getAbsolutePath(), name);
+ }
+ else {
+ throw new RuntimeException("Sharelib dir not found: " + name);
+ }
+ }
+ return dir.getAbsolutePath();
+ }
+
+ private static String findClasspathFile(String sharelib) throws Exception {
+ String classpathFile = null;
+ String sharelibDir = findDir("sharelib");
+ if (sharelibDir != null) {
+ File file = new File(new File(new File(sharelibDir, sharelib), "target"), "classpath");
+ if (file.exists()) {
+ classpathFile = file.getAbsolutePath();
+ }
+ else {
+ throw new RuntimeException("Sharelib classpath file for '" + sharelib +
+ "' not found, Run 'mvn generate-test-resources' from Oozie source root");
+ }
+ }
+ return classpathFile;
+ }
+
+ private static String[] getSharelibJars(String sharelib) throws Exception {
+ String classpathFile = findClasspathFile(sharelib);
+ BufferedReader br = new BufferedReader(new FileReader(classpathFile));
+ String line = br.readLine();
+ br.close();
+ return line.split(System.getProperty("path.separator"));
+ }
+
+ private static Path[] copySharelibJarsToFileSytem(String sharelib, FileSystem fs, Path targetDir) throws Exception {
+ String[] jars = getSharelibJars(sharelib);
+ List<Path> paths = new ArrayList<Path>();
+ for (String jar : jars) {
+ if (jar.endsWith(".jar")) {
+ Path targetPath = new Path(targetDir, new File(jar).getName());
+ InputStream is = new FileInputStream(jar);
+ OutputStream os = fs.create(targetPath);
+ IOUtils.copyStream(is, os);
+ paths.add(targetPath);
+ }
+ }
+ return paths.toArray(new Path[paths.size()]);
+ }
+
+ public static void addToDistributedCache(String sharelib, FileSystem fs, Path targetDir,
+ Configuration conf) throws Exception {
+ Path[] paths = copySharelibJarsToFileSytem(sharelib, fs, targetDir);
+ for (Path path : paths) {
+ DistributedCache.addFileToClassPath(path, conf, fs);
+ }
+ }
+
+}
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java?rev=1292481&r1=1292480&r2=1292481&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java Wed Feb 22 19:55:51 2012
@@ -25,29 +25,11 @@ import java.io.OutputStreamWriter;
import java.io.StringReader;
import java.io.Writer;
import java.text.MessageFormat;
-import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.Properties;
-
-import javax.jdo.Query;
-
-
-import org.antlr.runtime.ANTLRFileStream;
-import org.apache.commons.collections.Bag;
-import org.apache.commons.lang.text.StrMatcher;
-import org.apache.derby.iapi.services.io.DerbyIOException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.contrib.serde2.RegexSerDe;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
-import org.apache.hadoop.hive.ql.exec.ExecDriver;
-import org.apache.hadoop.hive.serde2.SerDe;
-import org.apache.hadoop.hive.service.HiveServer;
-import org.apache.hadoop.hive.shims.Hadoop20Shims;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobID;
@@ -63,15 +45,8 @@ import org.apache.oozie.util.ClassUtils;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XmlUtils;
-import org.apache.thrift.TBase;
-import org.datanucleus.FetchGroup;
-import org.datanucleus.store.rdbms.RDBMSStoreManager;
import org.jdom.Element;
import org.jdom.Namespace;
-import org.json.JSONML;
-import org.objectweb.asm.ByteVector;
-
-import com.facebook.fb303.FacebookBase;
public class TestHiveActionExecutor extends ActionExecutorTestCase {
@@ -133,11 +108,11 @@ public class TestHiveActionExecutor exte
"<configuration>" +
"<property>" +
"<name>javax.jdo.option.ConnectionURL</name>" +
- "<value>jdbc:hsqldb:mem:hive-main;create=true</value>" +
+ "<value>jdbc:derby:db;create=true</value>" +
"</property>" +
"<property>" +
"<name>javax.jdo.option.ConnectionDriverName</name>" +
- "<value>org.hsqldb.jdbcDriver</value>" +
+ "<value>org.apache.derby.jdbc.EmbeddedDriver</value>" +
"</property>" +
"<property>" +
"<name>javax.jdo.option.ConnectionUserName</name>" +
@@ -148,6 +123,10 @@ public class TestHiveActionExecutor exte
"<value> </value>" +
"</property>" +
"<property>" +
+ "<name>oozie.hive.log.level</name>" +
+ "<value>DEBUG</value>" +
+ "</property>" +
+ "<property>" +
"<name>oozie.hive.defaults</name>" +
"<value>user-hive-default.xml</value>" +
"</property>" +
@@ -229,7 +208,8 @@ public class TestHiveActionExecutor exte
conf.set("user.name", context.getProtoActionConf().get("user.name"));
conf.set("group.name", getTestGroup());
injectKerberosInfo(conf);
- JobConf jobConf = new JobConf(conf);
+ JobConf jobConf = new JobConf();
+ XConfiguration.copy(conf, jobConf);
String user = jobConf.get("user.name");
String group = jobConf.get("group.name");
JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, group, jobConf);
@@ -252,38 +232,14 @@ public class TestHiveActionExecutor exte
}
private Context createContext(String actionXml) throws Exception {
- List<String> jars = new ArrayList<String>();
HiveActionExecutor ae = new HiveActionExecutor();
- jars.add(copyJar("lib/libfb303-x.jar", FacebookBase.class));
- jars.add(copyJar("lib/libthrift-x.jar", TBase.class));
- jars.add(copyJar("lib/libjson-x.jar", JSONML.class));
- jars.add(copyJar("lib/jdo2-api-x.jar", Query.class));
- jars.add(copyJar("lib/antlr-runtime-x.jar", ANTLRFileStream.class));
- jars.add(copyJar("lib/asm-x.jar", ByteVector.class));
- jars.add(copyJar("lib/commons-collection-x.jar", Bag.class));
- jars.add(copyJar("lib/commons-lang-x.jar", StrMatcher.class));
- jars.add(copyJar("lib/datanucleus-core-x.jar", FetchGroup.class));
- jars.add(copyJar("lib/datanucleus-rdbms-x.jar", RDBMSStoreManager.class));
- jars.add(copyJar("lib/jdo-x.jar", javax.jdo.metadata.TypeMetadata.class));
- jars.add(copyJar("lib/derby.jar", DerbyIOException.class));
- jars.add(copyJar("lib/jline.jar", jline.FileNameCompletor.class));
- jars.add(copyJar("lib/hive-cli-x.jar", CliSessionState.class));
- jars.add(copyJar("lib/hive-common-x.jar", HiveConf.class));
- jars.add(copyJar("lib/hive-exec-x.jar", ExecDriver.class));
- jars.add(copyJar("lib/hive-metastore-x.jar", HiveMetaStore.class));
- jars.add(copyJar("lib/hive-serde-x.jar", SerDe.class));
- jars.add(copyJar("lib/hive-service-x.jar", HiveServer.class));
- jars.add(copyJar("lib/hive-shims-x.jar", Hadoop20Shims.class));
- jars.add(copyJar("lib/hive-contrib-x.jar", RegexSerDe.class));
-
XConfiguration protoConf = new XConfiguration();
protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
protoConf.set(WorkflowAppService.HADOOP_UGI, getTestUser() + "," + getTestGroup());
protoConf.set(OozieClient.GROUP_NAME, getTestGroup());
injectKerberosInfo(protoConf);
- protoConf.setStrings(WorkflowAppService.APP_LIB_PATH_LIST,
- jars.toArray(new String[jars.size()]));
+ SharelibUtils.addToDistributedCache("hive", getFileSystem(), getFsTestCaseDir(), protoConf);
WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive-action");
WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java?rev=1292481&r1=1292480&r2=1292481&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java Wed Feb 22 19:55:51 2012
@@ -17,34 +17,11 @@
*/
package org.apache.oozie.action.hadoop;
-import com.facebook.fb303.FacebookBase;
-import org.antlr.runtime.ANTLRFileStream;
-import org.apache.commons.collections.Bag;
-import org.apache.commons.lang.text.StrMatcher;
-import org.apache.derby.iapi.services.io.DerbyIOException;
-import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.contrib.serde2.RegexSerDe;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
-import org.apache.hadoop.hive.ql.exec.ExecDriver;
-import org.apache.hadoop.hive.serde2.SerDe;
-import org.apache.hadoop.hive.service.HiveServer;
-import org.apache.hadoop.hive.shims.Hadoop20Shims;
-import org.apache.oozie.util.ClassUtils;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XConfiguration;
-import org.apache.thrift.TBase;
-import org.datanucleus.FetchGroup;
-import org.datanucleus.store.rdbms.RDBMSStoreManager;
-import org.json.JSONML;
-import org.objectweb.asm.ByteVector;
-
-import javax.jdo.Query;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.InputStream;
@@ -52,8 +29,6 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
public class TestHiveMain extends MainTestCase {
private SecurityManager SECURITY_MANAGER;
@@ -84,19 +59,6 @@ public class TestHiveMain extends MainTe
return buffer.toString();
}
- private String copyJar(String targetFile, Class<?> anyContainedClass)
- throws Exception {
- String file = ClassUtils.findContainingJar(anyContainedClass);
- System.out.println("[copy-jar] class: " + anyContainedClass
- + ", local jar ==> " + file);
- Path targetPath = new Path(getFsTestCaseDir(), targetFile);
- FileSystem fs = getFileSystem();
- InputStream is = new FileInputStream(file);
- OutputStream os = fs.create(targetPath);
- IOUtils.copyStream(is, os);
- return targetPath.toString();
- }
-
public Void call() throws Exception {
if (System.getenv("HADOOP_HOME") == null) {
System.out.println("WARNING: 'HADOOP_HOME' env var not defined, TestHiveMain test is not running");
@@ -104,30 +66,6 @@ public class TestHiveMain extends MainTe
else {
FileSystem fs = getFileSystem();
- List<String> jars = new ArrayList<String>();
- jars.add(copyJar("lib/libfb303-x.jar", FacebookBase.class));
- jars.add(copyJar("lib/libthrift-x.jar", TBase.class));
- jars.add(copyJar("lib/libjson-x.jar", JSONML.class));
- jars.add(copyJar("lib/jdo2-api-x.jar", Query.class));
- jars.add(copyJar("lib/antlr-runtime-x.jar", ANTLRFileStream.class));
- jars.add(copyJar("lib/asm-x.jar", ByteVector.class));
- jars.add(copyJar("lib/commons-collection-x.jar", Bag.class));
- jars.add(copyJar("lib/commons-lang-x.jar", StrMatcher.class));
- jars.add(copyJar("lib/datanucleus-core-x.jar", FetchGroup.class));
- jars.add(copyJar("lib/datanucleus-rdbms-x.jar", RDBMSStoreManager.class));
- jars.add(copyJar("lib/jdo-x.jar", javax.jdo.metadata.TypeMetadata.class));
- jars.add(copyJar("lib/derby.jar", DerbyIOException.class));
- jars.add(copyJar("lib/jline.jar", jline.FileNameCompletor.class));
-
- jars.add(copyJar("lib/hive-cli-x.jar", CliSessionState.class));
- jars.add(copyJar("lib/hive-common-x.jar", HiveConf.class));
- jars.add(copyJar("lib/hive-exec-x.jar", ExecDriver.class));
- jars.add(copyJar("lib/hive-metastore-x.jar", HiveMetaStore.class));
- jars.add(copyJar("lib/hive-serde-x.jar", SerDe.class));
- jars.add(copyJar("lib/hive-service-x.jar", HiveServer.class));
- jars.add(copyJar("lib/hive-shims-x.jar", Hadoop20Shims.class));
- jars.add(copyJar("lib/hive-contrib-x.jar", RegexSerDe.class));
-
Path inputDir = new Path(getFsTestCaseDir(), "input");
fs.mkdirs(inputDir);
Writer writer = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
@@ -143,6 +81,8 @@ public class TestHiveMain extends MainTe
XConfiguration jobConf = new XConfiguration();
+ jobConf.set("oozie.hive.log.level", "DEBUG");
+
jobConf.set("user.name", getTestUser());
jobConf.set("group.name", getTestGroup());
jobConf.setInt("mapred.map.tasks", 1);
@@ -150,15 +90,12 @@ public class TestHiveMain extends MainTe
jobConf.setInt("mapred.reduce.max.attempts", 1);
jobConf.set("mapred.job.tracker", getJobTrackerUri());
jobConf.set("fs.default.name", getNameNodeUri());
- jobConf.set("javax.jdo.option.ConnectionURL", "jdbc:hsqldb:mem:hive-main;create=true");
- jobConf.set("javax.jdo.option.ConnectionDriverName", "org.hsqldb.jdbcDriver");
+ jobConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:db;create=true");
+ jobConf.set("javax.jdo.option.ConnectionDriverName", "org.apache.derby.jdbc.EmbeddedDriver");
jobConf.set("javax.jdo.option.ConnectionUserName", "sa");
+ jobConf.set("javax.jdo.option.ConnectionPassword", " ");
injectKerberosInfo(jobConf);
-
- for (String jar : jars) {
- DistributedCache.addFileToClassPath(new Path(new Path(jar).toUri().getPath()),
- getFileSystem().getConf());
- }
+ SharelibUtils.addToDistributedCache("hive", fs, getFsTestCaseDir(), jobConf);
HiveMain.setHiveScript(jobConf, script.toString(), new String[]{"IN=" + inputDir.toUri().getPath(),
"OUT=" + outputDir.toUri().getPath()});
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java?rev=1292481&r1=1292480&r2=1292481&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java Wed Feb 22 19:55:51 2012
@@ -34,8 +34,6 @@ import org.apache.oozie.service.HadoopAc
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XmlUtils;
import org.apache.oozie.util.IOUtils;
-import org.apache.oozie.util.ClassUtils;
-import org.apache.pig.Main;
import org.jdom.Element;
import org.json.simple.JSONValue;
@@ -50,9 +48,6 @@ import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
-
-import jline.ConsoleReaderInputStream;
public class TestPigActionExecutor extends ActionExecutorTestCase {
@@ -137,22 +132,13 @@ public class TestPigActionExecutor exten
FileSystem fs = getFileSystem();
- Path pigJar = new Path(getAppPath(), "lib/pig.jar");
- InputStream is = new FileInputStream(ClassUtils.findContainingJar(Main.class));
- OutputStream os = fs.create(new Path(getAppPath(), pigJar));
- IOUtils.copyStream(is, os);
-
- Path jLineJar = new Path(getAppPath(), "lib/jline.jar");
- is = new FileInputStream(ClassUtils.findContainingJar(ConsoleReaderInputStream.class));
- os = fs.create(new Path(getAppPath(), jLineJar));
- IOUtils.copyStream(is, os);
-
XConfiguration protoConf = new XConfiguration();
protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
protoConf.set(WorkflowAppService.HADOOP_UGI, getTestUser() + "," + getTestGroup());
protoConf.set(OozieClient.GROUP_NAME, getTestGroup());
injectKerberosInfo(protoConf);
- protoConf.setStrings(WorkflowAppService.APP_LIB_PATH_LIST, pigJar.toString(), jLineJar.toString());
+
+ SharelibUtils.addToDistributedCache("pig", fs, getFsTestCaseDir(), protoConf);
WorkflowJobBean wf = createBaseWorkflow(protoConf, "pig-action");
WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
@@ -182,10 +168,12 @@ public class TestPigActionExecutor exten
new XConfiguration(new StringReader(XmlUtils.prettyPrint(e.getChild("configuration")).toString()));
conf.set("mapred.job.tracker", e.getChildTextTrim("job-tracker"));
conf.set("fs.default.name", e.getChildTextTrim("name-node"));
+ conf.set("mapreduce.framework.name", "yarn");
conf.set("user.name", context.getProtoActionConf().get("user.name"));
conf.set("group.name", getTestGroup());
injectKerberosInfo(conf);
- JobConf jobConf = new JobConf(conf);
+ JobConf jobConf = new JobConf();
+ XConfiguration.copy(conf, jobConf);
String user = jobConf.get("user.name");
String group = jobConf.get("group.name");
JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, group, jobConf);
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java?rev=1292481&r1=1292480&r2=1292481&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java Wed Feb 22 19:55:51 2012
@@ -19,17 +19,12 @@ package org.apache.oozie.action.hadoop;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.filecache.DistributedCache;
import org.apache.oozie.util.XConfiguration;
-import org.apache.oozie.util.ClassUtils;
import org.apache.oozie.util.IOUtils;
-import org.apache.pig.Main;
import org.json.simple.JSONValue;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
-import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
@@ -39,8 +34,6 @@ import java.util.Map;
import java.util.Properties;
import java.net.URL;
-import jline.ConsoleReaderInputStream;
-
public class TestPigMain extends PigTestCase {
private SecurityManager SECURITY_MANAGER;
@@ -60,16 +53,6 @@ public class TestPigMain extends PigTest
public Void call() throws Exception {
FileSystem fs = getFileSystem();
- Path pigJar = new Path(getFsTestCaseDir(), "pig.jar");
- InputStream is = new FileInputStream(ClassUtils.findContainingJar(Main.class));
- OutputStream os = fs.create(pigJar);
- IOUtils.copyStream(is, os);
-
- Path jlineJar = new Path(getFsTestCaseDir(), "jline.jar");
- is = new FileInputStream(ClassUtils.findContainingJar(ConsoleReaderInputStream.class));
- os = fs.create(jlineJar);
- IOUtils.copyStream(is, os);
-
Path script = new Path(getTestCaseDir(), "script.pig");
Writer w = new FileWriter(script.toString());
w.write(pigScript);
@@ -93,19 +76,20 @@ public class TestPigMain extends PigTest
jobConf.set("mapred.job.tracker", getJobTrackerUri());
jobConf.set("fs.default.name", getNameNodeUri());
+ jobConf.set("mapreduce.framework.name", "yarn");
+
// option to specify whether stats should be stored or not
jobConf.set("oozie.action.external.stats.write", Boolean.toString(writeStats));
injectKerberosInfo(jobConf);
- DistributedCache.addFileToClassPath(new Path(pigJar.toUri().getPath()), getFileSystem().getConf());
- DistributedCache.addFileToClassPath(new Path(jlineJar.toUri().getPath()), getFileSystem().getConf());
+ SharelibUtils.addToDistributedCache("pig", fs, getFsTestCaseDir(), jobConf);
PigMain.setPigScript(jobConf, script.toString(), new String[] { "IN=" + inputDir.toUri().getPath(),
"OUT=" + outputDir.toUri().getPath() }, new String[] { "-v" });
File actionXml = new File(getTestCaseDir(), "action.xml");
- os = new FileOutputStream(actionXml);
+ OutputStream os = new FileOutputStream(actionXml);
jobConf.writeXml(os);
os.close();
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java?rev=1292481&r1=1292480&r2=1292481&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java Wed Feb 22 19:55:51 2012
@@ -32,11 +32,9 @@ import org.apache.oozie.client.WorkflowA
import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.WorkflowAppService;
-import org.apache.oozie.util.ClassUtils;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XmlUtils;
-import org.apache.sqoop.Sqoop;
import org.jdom.Element;
import org.jdom.Namespace;
@@ -270,9 +268,11 @@ public class TestSqoopActionExecutor ext
conf.set("mapred.job.tracker", e.getChildTextTrim("job-tracker", ns));
conf.set("fs.default.name", e.getChildTextTrim("name-node", ns));
conf.set("user.name", context.getProtoActionConf().get("user.name"));
+ conf.set("mapreduce.framework.name", "yarn");
conf.set("group.name", getTestGroup());
injectKerberosInfo(conf);
- JobConf jobConf = new JobConf(conf);
+ JobConf jobConf = new JobConf();
+ XConfiguration.copy(conf, jobConf);
String user = jobConf.get("user.name");
String group = jobConf.get("group.name");
JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, group, jobConf);
@@ -281,35 +281,19 @@ public class TestSqoopActionExecutor ext
return runningJob;
}
- private String copyJar(String targetFile, Class<?> anyContainedClass)
- throws Exception {
- String file = ClassUtils.findContainingJar(anyContainedClass);
- Path targetPath = new Path(getAppPath(), targetFile);
- FileSystem fs = getFileSystem();
- InputStream is = new FileInputStream(file);
- OutputStream os = fs.create(new Path(getAppPath(), targetPath));
- IOUtils.copyStream(is, os);
- return targetPath.toString();
- }
-
private Context createContext(String actionXml) throws Exception {
- List<String> jars = new ArrayList<String>();
SqoopActionExecutor ae = new SqoopActionExecutor();
- jars.add(copyJar("lib/commons-io-x.jar", org.apache.commons.io.IOUtils.class));
- jars.add(copyJar("lib/hsqldb-x.jar", org.hsqldb.jdbcDriver.class));
- jars.add(copyJar("lib/sqoop-x.jar", Sqoop.class));
-
XConfiguration protoConf = new XConfiguration();
protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
protoConf.set(WorkflowAppService.HADOOP_UGI, getTestUser() + "," + getTestGroup());
protoConf.set(OozieClient.GROUP_NAME, getTestGroup());
injectKerberosInfo(protoConf);
- //Making DB file to show up in the DistributeCache
- jars.addAll(copyDbToHdfs());
+ FileSystem fs = getFileSystem();
+ SharelibUtils.addToDistributedCache("sqoop", fs, getFsTestCaseDir(), protoConf);
- protoConf.setStrings(WorkflowAppService.APP_LIB_PATH_LIST, jars.toArray(new String[jars.size()]));
+ protoConf.setStrings(WorkflowAppService.APP_LIB_PATH_LIST, copyDbToHdfs());
WorkflowJobBean wf = createBaseWorkflow(protoConf, "sqoop-action");
WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
@@ -319,7 +303,7 @@ public class TestSqoopActionExecutor ext
return new Context(wf, action);
}
- private List<String> copyDbToHdfs() throws Exception {
+ private String[] copyDbToHdfs() throws Exception {
List<String> files = new ArrayList<String>();
String[] exts = {".script", ".properties"};
for (String ext : exts) {
@@ -332,6 +316,6 @@ public class TestSqoopActionExecutor ext
IOUtils.copyStream(is, os);
files.add(targetPath.toString() + "#" + name);
}
- return files;
+ return files.toArray(new String[files.size()]);
}
}
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java?rev=1292481&r1=1292480&r2=1292481&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java Wed Feb 22 19:55:51 2012
@@ -19,11 +19,7 @@ package org.apache.oozie.action.hadoop;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.streaming.StreamJob;
-import org.apache.hadoop.filecache.DistributedCache;
import org.apache.oozie.util.XConfiguration;
-import org.apache.oozie.util.ClassUtils;
-import org.apache.oozie.util.IOUtils;
import java.io.File;
import java.io.FileInputStream;
@@ -39,11 +35,6 @@ public class TestStreamingMain extends M
public Void call() throws Exception {
FileSystem fs = getFileSystem();
- Path streamingJar = new Path(getFsTestCaseDir(), "hadoop-streaming.jar");
- InputStream is = new FileInputStream(ClassUtils.findContainingJar(StreamJob.class));
- OutputStream os = fs.create(streamingJar);
- IOUtils.copyStream(is, os);
-
Path inputDir = new Path(getFsTestCaseDir(), "input");
fs.mkdirs(inputDir);
Writer writer = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
@@ -64,7 +55,7 @@ public class TestStreamingMain extends M
jobConf.set("user.name", getTestUser());
jobConf.set("hadoop.job.ugi", getTestUser() + "," + getTestGroup());
- DistributedCache.addFileToClassPath(new Path(streamingJar.toUri().getPath()), fs.getConf());
+ SharelibUtils.addToDistributedCache("streaming", fs, getFsTestCaseDir(), jobConf);
StreamingMain.setStreaming(jobConf, "cat", "wc", null, null, null);
@@ -72,7 +63,7 @@ public class TestStreamingMain extends M
jobConf.set("mapred.output.dir", outputDir.toString());
File actionXml = new File(getTestCaseDir(), "action.xml");
- os = new FileOutputStream(actionXml);
+ OutputStream os = new FileOutputStream(actionXml);
jobConf.writeXml(os);
os.close();
@@ -85,7 +76,7 @@ public class TestStreamingMain extends M
assertTrue(newIdProperties.exists());
- is = new FileInputStream(newIdProperties);
+ InputStream is = new FileInputStream(newIdProperties);
Properties props = new Properties();
props.load(is);
is.close();
Modified: incubator/oozie/trunk/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/pom.xml?rev=1292481&r1=1292480&r2=1292481&view=diff
==============================================================================
--- incubator/oozie/trunk/pom.xml (original)
+++ incubator/oozie/trunk/pom.xml Wed Feb 22 19:55:51 2012
@@ -431,6 +431,11 @@
<artifactId>slf4j-log4j12</artifactId>
<version>1.5.8</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-builtins</artifactId>
+ <version>${hive.version}</version>
+ </dependency>
<dependency>
<groupId>commons-logging</groupId>
@@ -679,6 +684,11 @@
<artifactId>maven-deploy-plugin</artifactId>
<version>2.5</version>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.4</version>
+ </plugin>
</plugins>
</pluginManagement>
Modified: incubator/oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1292481&r1=1292480&r2=1292481&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Wed Feb 22 19:55:51 2012
@@ -1,5 +1,6 @@
-- Oozie 3.2.0 release
+OOZIE-704 Sharelib action JARs for testcases should be collected leveraging Maven dependency information (tucu)
OOZIE-703 Improve/Consolidate Hadoop job ID log harvesting logic (tucu)
OOZIE-702 XTestCase minicluster fails with Hadoop 1.x (tucu)
OOZIE-700 update hadooplibs versions (tucu)
Modified: incubator/oozie/trunk/sharelib/hive/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/sharelib/hive/pom.xml?rev=1292481&r1=1292480&r2=1292481&view=diff
==============================================================================
--- incubator/oozie/trunk/sharelib/hive/pom.xml (original)
+++ incubator/oozie/trunk/sharelib/hive/pom.xml Wed Feb 22 19:55:51 2012
@@ -100,6 +100,11 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-builtins</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr-runtime</artifactId>
<version>3.0.1</version>
@@ -115,6 +120,23 @@
</resource>
</resources>
<plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>gen-classpath</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <includeScope>compile</includeScope>
+ <outputFile>${project.build.directory}/classpath</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<!--<plugin>-->
<!--<groupId>org.apache.maven.plugins</groupId>-->
<!--<artifactId>maven-deploy-plugin</artifactId>-->
Modified: incubator/oozie/trunk/sharelib/oozie/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/sharelib/oozie/pom.xml?rev=1292481&r1=1292480&r2=1292481&view=diff
==============================================================================
--- incubator/oozie/trunk/sharelib/oozie/pom.xml (original)
+++ incubator/oozie/trunk/sharelib/oozie/pom.xml Wed Feb 22 19:55:51 2012
@@ -54,6 +54,23 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>gen-classpath</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <includeScope>compile</includeScope>
+ <outputFile>${project.build.directory}/classpath</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<finalName>partial-sharelib</finalName>
Modified: incubator/oozie/trunk/sharelib/pig/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/sharelib/pig/pom.xml?rev=1292481&r1=1292480&r2=1292481&view=diff
==============================================================================
--- incubator/oozie/trunk/sharelib/pig/pom.xml (original)
+++ incubator/oozie/trunk/sharelib/pig/pom.xml Wed Feb 22 19:55:51 2012
@@ -75,6 +75,23 @@
</resource>
</resources>
<plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>gen-classpath</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <includeScope>compile</includeScope>
+ <outputFile>${project.build.directory}/classpath</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<!--<plugin>-->
<!--<groupId>org.apache.maven.plugins</groupId>-->
<!--<artifactId>maven-deploy-plugin</artifactId>-->
Modified: incubator/oozie/trunk/sharelib/sqoop/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/sharelib/sqoop/pom.xml?rev=1292481&r1=1292480&r2=1292481&view=diff
==============================================================================
--- incubator/oozie/trunk/sharelib/sqoop/pom.xml (original)
+++ incubator/oozie/trunk/sharelib/sqoop/pom.xml Wed Feb 22 19:55:51 2012
@@ -185,6 +185,23 @@
</resource>
</resources>
<plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>gen-classpath</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <includeScope>compile</includeScope>
+ <outputFile>${project.build.directory}/classpath</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<!--<plugin>-->
<!--<groupId>org.apache.maven.plugins</groupId>-->
<!--<artifactId>maven-deploy-plugin</artifactId>-->
Modified: incubator/oozie/trunk/sharelib/streaming/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/sharelib/streaming/pom.xml?rev=1292481&r1=1292480&r2=1292481&view=diff
==============================================================================
--- incubator/oozie/trunk/sharelib/streaming/pom.xml (original)
+++ incubator/oozie/trunk/sharelib/streaming/pom.xml Wed Feb 22 19:55:51 2012
@@ -55,6 +55,23 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>gen-classpath</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <includeScope>compile</includeScope>
+ <outputFile>${project.build.directory}/classpath</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<finalName>partial-sharelib</finalName>