You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2013/02/04 19:03:10 UTC
svn commit: r1442257 - in /oozie/branches/branch-3.3: ./ core/
core/src/test/java/org/apache/oozie/action/hadoop/
core/src/test/java/org/apache/oozie/test/ docs/src/site/twiki/ tests/
tests/pig/ tests/pig/src/ tests/pig/src/test/ tests/pig/src/test/jav...
Author: rkanter
Date: Mon Feb 4 18:03:09 2013
New Revision: 1442257
URL: http://svn.apache.org/viewvc?rev=1442257&view=rev
Log:
OOZIE-1124 Split pig unit tests to a separate module (rohini via virag)
Added:
oozie/branches/branch-3.3/tests/
oozie/branches/branch-3.3/tests/pig/
oozie/branches/branch-3.3/tests/pig/pom.xml
oozie/branches/branch-3.3/tests/pig/src/
oozie/branches/branch-3.3/tests/pig/src/test/
oozie/branches/branch-3.3/tests/pig/src/test/java/
oozie/branches/branch-3.3/tests/pig/src/test/java/org/
oozie/branches/branch-3.3/tests/pig/src/test/java/org/apache/
oozie/branches/branch-3.3/tests/pig/src/test/java/org/apache/oozie/
oozie/branches/branch-3.3/tests/pig/src/test/java/org/apache/oozie/action/
oozie/branches/branch-3.3/tests/pig/src/test/java/org/apache/oozie/action/hadoop/
oozie/branches/branch-3.3/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
oozie/branches/branch-3.3/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
oozie/branches/branch-3.3/tests/pom.xml
Removed:
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
Modified:
oozie/branches/branch-3.3/core/pom.xml
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/test/XTestCase.java
oozie/branches/branch-3.3/docs/src/site/twiki/ENG_Building.twiki
oozie/branches/branch-3.3/pom.xml
oozie/branches/branch-3.3/release-log.txt
Modified: oozie/branches/branch-3.3/core/pom.xml
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/pom.xml?rev=1442257&r1=1442256&r2=1442257&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/pom.xml (original)
+++ oozie/branches/branch-3.3/core/pom.xml Mon Feb 4 18:03:09 2013
@@ -162,11 +162,11 @@
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <scope>compile</scope>
- </dependency>
+ <dependency>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ <scope>compile</scope>
+ </dependency>
<!--
Oozie web-app module must exclude it.
@@ -224,9 +224,91 @@
</dependency>
<dependency>
+ <groupId>org.apache.pig</groupId>
+ <artifactId>pig</artifactId>
+ <version>0.8.0</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-api-2.1</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-2.1</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-el</groupId>
+ <artifactId>commons-el</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.sf.kosmosfs</groupId>
+ <artifactId>kfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>oro</groupId>
+ <artifactId>oro</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr-runtime</artifactId>
+ <version>3.0.1</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.oozie</groupId>
<artifactId>oozie-sharelib-pig</artifactId>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.pig</groupId>
+ <artifactId>pig</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr-runtime</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
@@ -465,7 +547,6 @@
<exclude>**/TestRerun.java</exclude>
<exclude>**/TestCallableQueueService.java</exclude>
<exclude>**/TestSsh*.java</exclude>
- <exclude>**/TestHive*.java</exclude>
<exclude>**/TestSqoop*.java</exclude>
<exclude>**/TestActionErrors.java</exclude>
<exclude>**/TestJavaActionExecutor.java</exclude>
@@ -582,27 +663,6 @@
</dependencies>
</profile>
- <!-- Forcing antlr-runtime 3.0.1 for Hive action testcases -->
- <!-- This is required because Pig 0.9.0 requires 3.4 and -->
- <!-- Hive 0.9.0 requires 3.0.1 -->
- <profile>
- <id>testHive</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>testHive</name>
- </property>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.antlr</groupId>
- <artifactId>antlr-runtime</artifactId>
- <version>3.0.1</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
-
<!-- Forcing Hadoop 0.20.2-cdh3u2 version because Sqoop -->
<!-- uses Hadoop API that is avail 0.21 onwards and the -->
<!-- Hadoop CDH version is the only stable version that -->
Modified: oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/test/XTestCase.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/test/XTestCase.java?rev=1442257&r1=1442256&r2=1442257&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/test/XTestCase.java (original)
+++ oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/test/XTestCase.java Mon Feb 4 18:03:09 2013
@@ -98,6 +98,10 @@ public abstract class XTestCase extends
OOZIE_SRC_DIR = new File(OOZIE_SRC_DIR, "core");
}
if (!OOZIE_SRC_DIR.exists()) {
+ OOZIE_SRC_DIR = OOZIE_SRC_DIR.getParentFile().getParentFile();
+ OOZIE_SRC_DIR = new File(OOZIE_SRC_DIR, "core");
+ }
+ if (!OOZIE_SRC_DIR.exists()) {
System.err.println();
System.err.println("Could not determine project root directory");
System.err.println();
Modified: oozie/branches/branch-3.3/docs/src/site/twiki/ENG_Building.twiki
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/docs/src/site/twiki/ENG_Building.twiki?rev=1442257&r1=1442256&r2=1442257&view=diff
==============================================================================
--- oozie/branches/branch-3.3/docs/src/site/twiki/ENG_Building.twiki (original)
+++ oozie/branches/branch-3.3/docs/src/site/twiki/ENG_Building.twiki Mon Feb 4 18:03:09 2013
@@ -197,15 +197,6 @@ file the following message 'SKIPPING TES
There are 2 testcases that use the *wordcount-simple* pipes binary, *TestPipesMain* and *TestMapReduceActionExecutor*,
the 'SKIPPING TEST..." message would appear in the testcase log file of both testcases.
----+++ Testing Hive Action
-
-Because of depedencies incompatibilities between Hive 0.9.0 and Pig 0.9.0 to run Hive testcases a special profile
-must be used together with the name of the Hive testcases.
-
-<verbatim>
-$ mvn -DtestHive -Dtest=TestHiveActionExecutor,TestHiveMain
-</verbatim>
-
---+++ Testing Sqoop Action
Apache Sqoop 1.5.0 requires functionality only available in Hadoop 0.21 and onwards. Currently, the only
@@ -217,10 +208,6 @@ purposes. *This is a temporary fix*.
$ mvn -DtestSqoop -Dtest=TestSqoopActionExecutor
</verbatim>
-<verbatim>
-$ mvn -DtestHive -Dtest=TestHiveActionExecutor,TestHiveMain
-</verbatim>
-
---++ Building an Oozie Distribution
An Oozie distribution bundles an embedded Tomcat server. The Oozie distro module downloads Tomcat TAR.GZ from Apache
Modified: oozie/branches/branch-3.3/pom.xml
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/pom.xml?rev=1442257&r1=1442256&r2=1442257&view=diff
==============================================================================
--- oozie/branches/branch-3.3/pom.xml (original)
+++ oozie/branches/branch-3.3/pom.xml Mon Feb 4 18:03:09 2013
@@ -95,6 +95,7 @@
<module>hadooplibs</module>
<module>hbaselibs</module>
<module>core</module>
+ <module>tests</module>
<module>webapp</module>
<module>examples</module>
<module>docs</module>
@@ -828,9 +829,6 @@
-->
<exclude>**/TestSsh*.java</exclude>
- <!-- See 'testHive' profile in core/pom.xml and the Building doc-->
- <exclude>**/TestHive*.java</exclude>
-
<!-- See 'testSqoop' profile in core/pom.xml and the Building doc-->
<exclude>**/TestSqoop*.java</exclude>
Modified: oozie/branches/branch-3.3/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/release-log.txt?rev=1442257&r1=1442256&r2=1442257&view=diff
==============================================================================
--- oozie/branches/branch-3.3/release-log.txt (original)
+++ oozie/branches/branch-3.3/release-log.txt Mon Feb 4 18:03:09 2013
@@ -1,5 +1,6 @@
-- Oozie 3.3.2 (unreleased)
+OOZIE-1124 Split pig unit tests to a separate module (rohini via virag)
OOZIE-1087 Remove requirement of hive-default.xml from Hive action (rkanter)
OOZIE-1129 Add documentation for configurable filesystem support (rkanter)
OOZIE-1084 When use IBM jdk , UT TestCallbackServlet and TestHadoopELFunctions fail (zhujinwei via rkanter)
Added: oozie/branches/branch-3.3/tests/pig/pom.xml
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/tests/pig/pom.xml?rev=1442257&view=auto
==============================================================================
--- oozie/branches/branch-3.3/tests/pig/pom.xml (added)
+++ oozie/branches/branch-3.3/tests/pig/pom.xml Mon Feb 4 18:03:09 2013
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-tests</artifactId>
+ <version>3.4.0-SNAPSHOT</version>
+ </parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-tests-pig</artifactId>
+ <version>3.4.0-SNAPSHOT</version>
+ <description>Apache Oozie Tests For Pig</description>
+ <name>Apache Oozie Tests For Pig</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <oozie.test.default.config.file>${basedir}/../../core/src/test/resources/${oozie.test.db}-oozie-site.xml
+ </oozie.test.default.config.file>
+ <oozie.test.config.file>${oozie.test.default.config.file}</oozie.test.config.file>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-pig</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ <resource>
+ <directory>${basedir}/../../core/src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ <testResource>
+ <directory>${basedir}/../../core/src/test/resources</directory>
+ </testResource>
+ </testResources>
+ </build>
+
+</project>
+
Added: oozie/branches/branch-3.3/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java?rev=1442257&view=auto
==============================================================================
--- oozie/branches/branch-3.3/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java (added)
+++ oozie/branches/branch-3.3/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java Mon Feb 4 18:03:09 2013
@@ -0,0 +1,416 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hadoop.mapred.JobID;
+import org.apache.oozie.WorkflowActionBean;
+import org.apache.oozie.WorkflowJobBean;
+import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.service.WorkflowAppService;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.service.HadoopAccessorService;
+import org.apache.oozie.util.XConfiguration;
+import org.apache.oozie.util.XmlUtils;
+import org.apache.oozie.util.IOUtils;
+import org.jdom.Element;
+import org.json.simple.JSONValue;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.InputStream;
+import java.io.FileInputStream;
+import java.io.Writer;
+import java.io.OutputStreamWriter;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class TestPigActionExecutor extends ActionExecutorTestCase {
+
+ private static final String PIG_SCRIPT = "set job.name 'test'\n" + "set debug on\n" +
+ "A = load '$IN' using PigStorage(':');\n" +
+ "B = foreach A generate $0 as id;\n" +
+ "store B into '$OUT' USING PigStorage();\n";
+
+ private static final String ERROR_PIG_SCRIPT = "set job.name 'test'\n" + "set debug on\n" +
+ "A = load '$IN' using PigStorage(':');\n" +
+ "ERROR @#$@#$;\n";
+
+ private static final String UDF_PIG_SCRIPT = "register udf.jar\n" +
+ "set job.name 'test'\n" + "set debug on\n" +
+ "A = load '$IN' using PigStorage(':');\n" +
+ "B = foreach A generate" +
+ " org.apache.oozie.action.hadoop.UDFTester($0) as id;\n" +
+ "store B into '$OUT' USING PigStorage();\n";
+
+ @Override
+ protected void setSystemProps() throws Exception {
+ super.setSystemProps();
+ setSystemProperty("oozie.service.ActionService.executor.classes", PigActionExecutor.class.getName());
+ }
+
+ public void testLauncherJar() throws Exception {
+ PigActionExecutor ae = new PigActionExecutor();
+ Path jar = new Path(ae.getOozieRuntimeDir(), ae.getLauncherJarName());
+ assertTrue(new File(jar.toString()).exists());
+ }
+
+ public void testSetupMethods() throws Exception {
+ PigActionExecutor ae = new PigActionExecutor();
+
+ assertEquals("pig", ae.getType());
+
+ assertEquals("pig-launcher.jar", ae.getLauncherJarName());
+
+ List<Class> classes = new ArrayList<Class>();
+ classes.add(LauncherMapper.class);
+ classes.add(LauncherSecurityManager.class);
+ classes.add(LauncherException.class);
+ classes.add(LauncherMainException.class);
+ classes.add(FileSystemActions.class);
+ classes.add(PrepareActionsDriver.class);
+ classes.add(ActionStats.class);
+ classes.add(ActionType.class);
+ classes.add(LauncherMain.class);
+ classes.add(MapReduceMain.class);
+ classes.add(PigMain.class);
+ classes.add(OoziePigStats.class);
+ assertEquals(classes, ae.getLauncherClasses());
+
+ Element actionXml = XmlUtils.parseXml("<pig>" +
+ "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
+ "<name-node>" + getNameNodeUri() + "</name-node>" +
+ "<script>SCRIPT</script>" +
+ "<param>a=A</param>" +
+ "<param>b=B</param>" +
+ "</pig>");
+
+ XConfiguration protoConf = new XConfiguration();
+ protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+
+
+ WorkflowJobBean wf = createBaseWorkflow(protoConf, "pig-action");
+ WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
+ action.setType(ae.getType());
+
+ Context context = new Context(wf, action);
+
+ Configuration conf = ae.createBaseHadoopConf(context, actionXml);
+ ae.setupActionConf(conf, context, actionXml, getFsTestCaseDir());
+ assertEquals("SCRIPT", conf.get("oozie.pig.script"));
+ assertEquals("2", conf.get("oozie.pig.params.size"));
+ assertEquals("a=A", conf.get("oozie.pig.params.0"));
+ assertEquals("b=B", conf.get("oozie.pig.params.1"));
+ }
+
+ private Context createContext(String actionXml) throws Exception {
+ PigActionExecutor ae = new PigActionExecutor();
+
+ FileSystem fs = getFileSystem();
+
+ XConfiguration protoConf = new XConfiguration();
+ protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+
+
+ SharelibUtils.addToDistributedCache("pig", fs, getFsTestCaseDir(), protoConf);
+
+ WorkflowJobBean wf = createBaseWorkflow(protoConf, "pig-action");
+ WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
+ action.setType(ae.getType());
+ action.setConf(actionXml);
+
+ return new Context(wf, action);
+ }
+
+ private RunningJob submitAction(Context context) throws Exception {
+ PigActionExecutor ae = new PigActionExecutor();
+
+ WorkflowAction action = context.getAction();
+
+ ae.prepareActionDir(getFileSystem(), context);
+ ae.submitLauncher(getFileSystem(), context, action);
+
+ String jobId = action.getExternalId();
+ String jobTracker = action.getTrackerUri();
+ String consoleUrl = action.getConsoleUrl();
+ assertNotNull(jobId);
+ assertNotNull(jobTracker);
+ assertNotNull(consoleUrl);
+
+ Element e = XmlUtils.parseXml(action.getConf());
+ XConfiguration conf =
+ new XConfiguration(new StringReader(XmlUtils.prettyPrint(e.getChild("configuration")).toString()));
+ conf.set("mapred.job.tracker", e.getChildTextTrim("job-tracker"));
+ conf.set("fs.default.name", e.getChildTextTrim("name-node"));
+ conf.set("mapreduce.framework.name", "yarn");
+ conf.set("user.name", context.getProtoActionConf().get("user.name"));
+ conf.set("group.name", getTestGroup());
+
+ JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker);
+ XConfiguration.copy(conf, jobConf);
+ String user = jobConf.get("user.name");
+ String group = jobConf.get("group.name");
+ JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, jobConf);
+ final RunningJob runningJob = jobClient.getJob(JobID.forName(jobId));
+ assertNotNull(runningJob);
+ return runningJob;
+ }
+
+ private void _testSubmit(String actionXml, boolean checkForSuccess) throws Exception {
+
+ Context context = createContext(actionXml);
+ final RunningJob launcherJob = submitAction(context);
+ String launcherId = context.getAction().getExternalId();
+ evaluateLauncherJob(launcherJob);
+ assertTrue(launcherJob.isSuccessful());
+
+ sleep(2000);
+ assertFalse(LauncherMapper.hasIdSwap(launcherJob));
+ if (checkForSuccess) {
+ assertTrue(LauncherMapper.hasStatsData(launcherJob));
+ }
+
+ PigActionExecutor ae = new PigActionExecutor();
+ ae.check(context, context.getAction());
+ ae.end(context, context.getAction());
+ assertTrue(launcherId.equals(context.getAction().getExternalId()));
+ if (checkForSuccess) {
+ assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
+ assertNull(context.getAction().getData());
+
+ }
+ else {
+ assertEquals("FAILED/KILLED", context.getAction().getExternalStatus());
+ assertNotNull(context.getAction().getErrorMessage());
+ }
+ if (checkForSuccess) {
+ assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
+ }
+ else {
+ assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus());
+ }
+ }
+
+ /*
+ * Test the stats retrieved from a Pig job
+ */
+ public void testExecutionStats() throws Exception {
+ // Set the action xml with the option for retrieving stats to true
+ String actionXml = setPigActionXml(PIG_SCRIPT, true);
+ Context context = createContext(actionXml);
+ final RunningJob launcherJob = submitAction(context);
+ evaluateLauncherJob(launcherJob);
+ assertTrue(launcherJob.isSuccessful());
+ assertTrue(LauncherMapper.hasStatsData(launcherJob));
+
+ PigActionExecutor ae = new PigActionExecutor();
+ WorkflowAction wfAction = context.getAction();
+ ae.check(context, wfAction);
+ ae.end(context, wfAction);
+
+ assertEquals("SUCCEEDED", wfAction.getExternalStatus());
+ String stats = wfAction.getStats();
+ assertNotNull(stats);
+ // check for some of the expected key values in the stats
+ Map m = (Map)JSONValue.parse(stats);
+ // check for expected 1st level JSON keys
+ assertTrue(m.containsKey("PIG_VERSION"));
+
+ String expectedChildIDs = wfAction.getExternalChildIDs();
+ String[] childIDs = expectedChildIDs.split(",");
+ assertTrue(m.containsKey(childIDs[0]));
+
+ Map q = (Map)m.get(childIDs[0]);
+ // check for expected 2nd level JSON keys
+ assertTrue(q.containsKey("HADOOP_COUNTERS"));
+ }
+
+
+ /*
+ * Test the Hadoop IDs obtained from the Pig job
+ */
+ public void testExternalChildIds() throws Exception {
+ // Set the action xml with the option for retrieving stats to false
+ String actionXml = setPigActionXml(PIG_SCRIPT, false);
+ Context context = createContext(actionXml);
+ final RunningJob launcherJob = submitAction(context);
+ evaluateLauncherJob(launcherJob);
+ assertTrue(launcherJob.isSuccessful());
+
+ PigActionExecutor ae = new PigActionExecutor();
+ WorkflowAction wfAction = context.getAction();
+ ae.check(context, wfAction);
+ ae.end(context, wfAction);
+
+ assertEquals("SUCCEEDED", wfAction.getExternalStatus());
+ String externalIds = wfAction.getExternalChildIDs();
+ assertNotNull(externalIds);
+ assertNotSame("", externalIds);
+ // check for the expected prefix of hadoop jobIDs
+ assertTrue(externalIds.contains("job_"));
+
+ }
+
+ /*
+ * Test the stats after setting the maximum allowed size of stats to a small
+ * value
+ */
+ public void testExecutionStatsWithMaxStatsSizeLimit() throws Exception {
+ Services.get().destroy();
+ // Set a very small value for max size of stats
+ setSystemProperty(JavaActionExecutor.MAX_EXTERNAL_STATS_SIZE, new String("1"));
+ new Services().init();
+ // Set the action xml with the option for retrieving stats to true
+ String actionXml = setPigActionXml(PIG_SCRIPT, true);
+ Context context = createContext(actionXml);
+ final RunningJob launcherJob = submitAction(context);
+ evaluateLauncherJob(launcherJob);
+ assertTrue(launcherJob.isSuccessful());
+
+ PigActionExecutor ae = new PigActionExecutor();
+ WorkflowAction wfAction = context.getAction();
+ ae.check(context, wfAction);
+ ae.end(context, wfAction);
+
+ // action should fail as the size of pig stats will always be greater
+ // than 1 byte
+ assertEquals("FAILED/KILLED", wfAction.getExternalStatus());
+ assertNull(wfAction.getStats());
+ }
+
+ /*
+ * Test the stats with retrieve stats option set to false
+ */
+ public void testExecutionStatsWithRetrieveStatsFalse() throws Exception {
+ // Set the action xml with the option for retrieving stats to false
+ String actionXml = setPigActionXml(PIG_SCRIPT, false);
+ Context context = createContext(actionXml);
+ final RunningJob launcherJob = submitAction(context);
+ evaluateLauncherJob(launcherJob);
+ assertTrue(launcherJob.isSuccessful());
+ assertFalse(LauncherMapper.hasStatsData(launcherJob));
+
+ PigActionExecutor ae = new PigActionExecutor();
+ WorkflowAction wfAction = context.getAction();
+ ae.check(context, wfAction);
+ ae.end(context, wfAction);
+
+ assertEquals("SUCCEEDED", wfAction.getExternalStatus());
+ assertNotNull(wfAction.getExternalChildIDs());
+ }
+
+ private void evaluateLauncherJob(final RunningJob launcherJob) throws Exception{
+ waitFor(180 * 1000, new Predicate() {
+ @Override
+ public boolean evaluate() throws Exception {
+ return launcherJob.isComplete();
+ }
+ });
+ sleep(2000);
+ }
+
+ protected XConfiguration setPigConfig(boolean writeStats) {
+ XConfiguration conf = new XConfiguration();
+ conf.set("oozie.pig.log.level", "INFO");
+ conf.set(PigMain.EXTERNAL_STATS_WRITE, String.valueOf(writeStats));
+ return conf;
+ }
+
+ public void testPig() throws Exception {
+ // Set the action xml with the option for retrieving stats to true
+ String actionXml = setPigActionXml(PIG_SCRIPT, true);
+ _testSubmit(actionXml, true);
+ }
+
+ public void testPigError() throws Exception {
+ // Set the action xml with the option for retrieving stats to true
+ String actionXml = setPigActionXml(ERROR_PIG_SCRIPT, true);
+ _testSubmit(actionXml, false);
+ }
+
+ private String setPigActionXml(String pigScript, boolean writeStats) throws IOException{
+ FileSystem fs = getFileSystem();
+
+ Path script = new Path(getAppPath(), "script.pig");
+ Writer w = new OutputStreamWriter(fs.create(script));
+ w.write(pigScript);
+ w.close();
+
+ Path inputDir = new Path(getFsTestCaseDir(), "input");
+ Path outputDir = new Path(getFsTestCaseDir(), "output");
+
+ w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
+ w.write("dummy\n");
+ w.write("dummy\n");
+ w.close();
+
+ String actionXml = "<pig>" +
+ "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
+ "<name-node>" + getNameNodeUri() + "</name-node>" +
+ setPigConfig(writeStats).toXmlString(false) +
+ "<script>" + script.getName() + "</script>" +
+ "<param>IN=" + inputDir.toUri().getPath() + "</param>" +
+ "<param>OUT=" + outputDir.toUri().getPath() + "</param>" +
+ "</pig>";
+
+ return actionXml;
+ }
+
+ public void testUdfPig() throws Exception {
+ FileSystem fs = getFileSystem();
+
+ Path udfJar = new Path(getFsTestCaseDir(), "udf.jar");
+ File jarFile = IOUtils.createJar(new File(getTestCaseDir()), "udf.jar", UDFTester.class);
+ InputStream is = new FileInputStream(jarFile);
+ OutputStream os = getFileSystem().create(udfJar);
+ IOUtils.copyStream(is, os);
+
+ Path script = new Path(getAppPath(), "script.pig");
+ Writer w = new OutputStreamWriter(fs.create(script));
+ w.write(UDF_PIG_SCRIPT);
+ w.close();
+
+ Path inputDir = new Path(getFsTestCaseDir(), "input");
+ Path outputDir = new Path(getFsTestCaseDir(), "output");
+
+ w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
+ w.write("dummy\n");
+ w.write("dummy\n");
+ w.close();
+
+ String actionXml = "<pig>" +
+ "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
+ "<name-node>" + getNameNodeUri() + "</name-node>" +
+ setPigConfig(true).toXmlString(false) +
+ "<script>" + script.getName() + "</script>" +
+ "<param>IN=" + inputDir.toUri().getPath() + "</param>" +
+ "<param>OUT=" + outputDir.toUri().getPath() + "</param>" +
+ "<file>" + udfJar.toString() + "#" + udfJar.getName() + "</file>" +
+ "</pig>";
+ _testSubmit(actionXml, true);
+ }
+
+}
Added: oozie/branches/branch-3.3/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java?rev=1442257&view=auto
==============================================================================
--- oozie/branches/branch-3.3/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java (added)
+++ oozie/branches/branch-3.3/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java Mon Feb 4 18:03:09 2013
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.util.XConfiguration;
+import org.apache.oozie.util.IOUtils;
+import org.json.simple.JSONValue;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.io.FileWriter;
+import java.io.FileReader;
+import java.util.Map;
+import java.util.Properties;
+import java.net.URL;
+
+public class TestPigMain extends PigTestCase {
+ private SecurityManager SECURITY_MANAGER;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ SECURITY_MANAGER = System.getSecurityManager();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ System.setSecurityManager(SECURITY_MANAGER);
+ super.tearDown();
+ }
+
+ @Override
+ public Void call() throws Exception {
+ FileSystem fs = getFileSystem();
+
+ Path script = new Path(getTestCaseDir(), "script.pig");
+ Writer w = new FileWriter(script.toString());
+ w.write(pigScript);
+ w.close();
+
+ Path inputDir = new Path(getFsTestCaseDir(), "input");
+ fs.mkdirs(inputDir);
+ Writer writer = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
+ writer.write("hello");
+ writer.close();
+
+ Path outputDir = new Path(getFsTestCaseDir(), "output");
+
+ XConfiguration jobConf = new XConfiguration();
+ XConfiguration.copy(createJobConf(), jobConf);
+
+ jobConf.set("user.name", getTestUser());
+ jobConf.set("group.name", getTestGroup());
+ jobConf.setInt("mapred.map.tasks", 1);
+ jobConf.setInt("mapred.map.max.attempts", 1);
+ jobConf.setInt("mapred.reduce.max.attempts", 1);
+
+ // option to specify whether stats should be stored or not
+ jobConf.set("oozie.action.external.stats.write", Boolean.toString(writeStats));
+
+
+
+ SharelibUtils.addToDistributedCache("pig", fs, getFsTestCaseDir(), jobConf);
+
+ PigMain.setPigScript(jobConf, script.toString(), new String[] { "IN=" + inputDir.toUri().getPath(),
+ "OUT=" + outputDir.toUri().getPath() }, new String[] { "-v" });
+
+ File actionXml = new File(getTestCaseDir(), "action.xml");
+ OutputStream os = new FileOutputStream(actionXml);
+ jobConf.writeXml(os);
+ os.close();
+
+ File statsDataFile = new File(getTestCaseDir(), "statsdata.properties");
+
+ File hadoopIdsFile = new File(getTestCaseDir(), "hadoopIds.properties");
+
+ setSystemProperty("oozie.launcher.job.id", "" + System.currentTimeMillis());
+ setSystemProperty("oozie.action.conf.xml", actionXml.getAbsolutePath());
+ setSystemProperty("oozie.action.stats.properties", statsDataFile.getAbsolutePath());
+ setSystemProperty("oozie.action.externalChildIDs.properties", hadoopIdsFile.getAbsolutePath());
+
+
+ URL url = Thread.currentThread().getContextClassLoader().getResource("PigMain.txt");
+ File classPathDir = new File(url.getPath()).getParentFile();
+ assertTrue(classPathDir.exists());
+ Properties props = jobConf.toProperties();
+ assertEquals(props.getProperty("oozie.pig.args.size"), "1");
+ File pigProps = new File(classPathDir, "pig.properties");
+
+ new LauncherSecurityManager();
+ String user = System.getProperty("user.name");
+ try {
+ Writer wr = new FileWriter(pigProps);
+ props.store(wr, "");
+ wr.close();
+ PigMain.main(null);
+ }
+ catch (SecurityException ex) {
+ if (LauncherSecurityManager.getExitInvoked()) {
+ System.out.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")");
+ System.err.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")");
+ if (LauncherSecurityManager.getExitCode() != 0) {
+ fail();
+ }
+ }
+ else {
+ throw ex;
+ }
+ }
+ finally {
+ pigProps.delete();
+ System.setProperty("user.name", user);
+ }
+
+ // Stats should be stored only if option to write stats is set to true
+ if (writeStats) {
+ assertTrue(statsDataFile.exists());
+ String stats = IOUtils.getReaderAsString(new FileReader(statsDataFile), -1);
+ // check for some of the expected key values in the stats
+ Map m = (Map) JSONValue.parse(stats);
+ // check for expected 1st level JSON keys
+ assertTrue(m.containsKey("PIG_VERSION"));
+ } else {
+ assertFalse(statsDataFile.exists());
+ }
+ // HadoopIds should always be stored
+ assertTrue(hadoopIdsFile.exists());
+ String externalChildIds = IOUtils.getReaderAsString(new FileReader(hadoopIdsFile), -1);
+ assertTrue(externalChildIds.contains("job_"));
+ return null;
+ }
+
+}
Added: oozie/branches/branch-3.3/tests/pom.xml
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/tests/pom.xml?rev=1442257&view=auto
==============================================================================
--- oozie/branches/branch-3.3/tests/pom.xml (added)
+++ oozie/branches/branch-3.3/tests/pom.xml Mon Feb 4 18:03:09 2013
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-main</artifactId>
+ <version>3.4.0-SNAPSHOT</version>
+ </parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-tests</artifactId>
+ <version>3.4.0-SNAPSHOT</version>
+ <description>Apache Oozie Tests</description>
+ <name>Apache Oozie Tests</name>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>pig</module>
+ </modules>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-core</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file