You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by an...@apache.org on 2012/01/06 22:57:19 UTC
svn commit: r1228428 - in /incubator/oozie/branches/3.1: core/pom.xml
core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
core/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java pom.xml
release-log.txt sharelib/pom.xml
Author: angeloh
Date: Fri Jan 6 21:57:18 2012
New Revision: 1228428
URL: http://svn.apache.org/viewvc?rev=1228428&view=rev
Log:
Closes OOZIE-564 Embedded pig within python fails when run using oozie
Modified:
incubator/oozie/branches/3.1/core/pom.xml
incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
incubator/oozie/branches/3.1/pom.xml
incubator/oozie/branches/3.1/release-log.txt
incubator/oozie/branches/3.1/sharelib/pom.xml
Modified: incubator/oozie/branches/3.1/core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/core/pom.xml?rev=1228428&r1=1228427&r2=1228428&view=diff
==============================================================================
--- incubator/oozie/branches/3.1/core/pom.xml (original)
+++ incubator/oozie/branches/3.1/core/pom.xml Fri Jan 6 21:57:18 2012
@@ -152,6 +152,24 @@
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.python</groupId>
+ <artifactId>jython</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr-runtime</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <scope>test</scope>
+ </dependency>
+
<!--
Oozie web-app module must exclude it.
-->
Modified: incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java?rev=1228428&r1=1228427&r2=1228428&view=diff
==============================================================================
--- incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java (original)
+++ incubator/oozie/branches/3.1/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java Fri Jan 6 21:57:18 2012
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -22,7 +22,6 @@ import org.apache.pig.PigRunner;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobClient;
import java.io.FileNotFoundException;
import java.io.OutputStream;
@@ -64,6 +63,7 @@ public class PigMain extends LauncherMai
run(PigMain.class, args);
}
+ @Override
protected void run(String[] args) throws Exception {
System.out.println();
System.out.println("Oozie Pig action configuration");
@@ -88,7 +88,7 @@ public class PigMain extends LauncherMai
pigProperties.setProperty(entry.getKey(), entry.getValue());
}
- //propagate delegation related props from launcher job to Pig job
+ // propagate delegation related props from launcher job to Pig job
if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
pigProperties.setProperty("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
System.out.println("------------------------");
@@ -241,6 +241,9 @@ public class PigMain extends LauncherMai
}
/**
+ * Runs the pig script using PigRunner API if version 0.8 or above. Embedded
+ * pig within python is also supported.
+ *
* @param args pig command line arguments
* @param pigLog pig log file
* @param resetSecurityManager specify if need to reset security manager
@@ -260,12 +263,13 @@ public class PigMain extends LauncherMai
if (pigRunnerExists) {
System.out.println("Run pig script using PigRunner.run() for Pig version 0.8+");
PigStats stats = PigRunner.run(args, null);
- int code = stats.getReturnCode();
- if (code != 0) {
+ // isSuccessful is the API from 0.9 supported by both PigStats and
+ // EmbeddedPigStats
+ if (!stats.isSuccessful()) {
if (pigLog != null) {
handleError(pigLog);
}
- throw new LauncherMainException(code);
+ throw new LauncherMainException(PigRunner.ReturnCode.FAILURE);
}
}
else {
Modified: incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java?rev=1228428&r1=1228427&r2=1228428&view=diff
==============================================================================
--- incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java (original)
+++ incubator/oozie/branches/3.1/core/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java Fri Jan 6 21:57:18 2012
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,7 +20,6 @@ package org.apache.oozie.action.hadoop;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.oozie.test.XFsTestCase;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.ClassUtils;
import org.apache.oozie.util.IOUtils;
@@ -36,31 +35,26 @@ import java.io.Writer;
import java.io.FileWriter;
import java.io.FileReader;
import java.util.Properties;
-import java.util.concurrent.Callable;
import java.net.URL;
import jline.ConsoleReaderInputStream;
-public class TestPigMain extends MainTestCase {
+public class TestPigMain extends PigTestCase {
private SecurityManager SECURITY_MANAGER;
+ @Override
protected void setUp() throws Exception {
super.setUp();
SECURITY_MANAGER = System.getSecurityManager();
}
+ @Override
protected void tearDown() throws Exception {
System.setSecurityManager(SECURITY_MANAGER);
super.tearDown();
}
- private static final String PIG_SCRIPT =
- "set job.name 'test'\n" +
- "set debug on\n" +
- "A = load '$IN' using PigStorage(':');\n" +
- "B = foreach A generate $0 as id;\n" +
- "store B into '$OUT' USING PigStorage();\n";
-
+ @Override
public Void call() throws Exception {
FileSystem fs = getFileSystem();
@@ -76,7 +70,7 @@ public class TestPigMain extends MainTes
Path script = new Path(getTestCaseDir(), "script.pig");
Writer w = new FileWriter(script.toString());
- w.write(PIG_SCRIPT);
+ w.write(pigScript);
w.close();
Path inputDir = new Path(getFsTestCaseDir(), "input");
@@ -101,8 +95,8 @@ public class TestPigMain extends MainTes
DistributedCache.addFileToClassPath(new Path(pigJar.toUri().getPath()), getFileSystem().getConf());
DistributedCache.addFileToClassPath(new Path(jlineJar.toUri().getPath()), getFileSystem().getConf());
- PigMain.setPigScript(jobConf, script.toString(), new String[]{"IN=" + inputDir.toUri().getPath(),
- "OUT=" + outputDir.toUri().getPath()}, new String[]{"-v"});
+ PigMain.setPigScript(jobConf, script.toString(), new String[] { "IN=" + inputDir.toUri().getPath(),
+ "OUT=" + outputDir.toUri().getPath() }, new String[] { "-v" });
File actionXml = new File(getTestCaseDir(), "action.xml");
os = new FileOutputStream(actionXml);
@@ -115,7 +109,6 @@ public class TestPigMain extends MainTes
setSystemProperty("oozie.action.conf.xml", actionXml.getAbsolutePath());
setSystemProperty("oozie.action.output.properties", outputDataFile.getAbsolutePath());
-
URL url = Thread.currentThread().getContextClassLoader().getResource("PigMain.txt");
File classPathDir = new File(url.getPath()).getParentFile();
assertTrue(classPathDir.exists());
Modified: incubator/oozie/branches/3.1/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/pom.xml?rev=1228428&r1=1228427&r2=1228428&view=diff
==============================================================================
--- incubator/oozie/branches/3.1/pom.xml (original)
+++ incubator/oozie/branches/3.1/pom.xml Fri Jan 6 21:57:18 2012
@@ -285,7 +285,7 @@
<!-- groupId to be correct by GH-0226 -->
<groupId>org.apache.pig</groupId>
<artifactId>pig</artifactId>
- <version>0.8.0</version>
+ <version>0.9.0</version>
<exclusions>
<exclusion>
<!-- groupId to be correct by GH-0226 -->
@@ -451,6 +451,24 @@
<version>9.0-801.jdbc4</version>
</dependency>
+ <dependency>
+ <groupId>org.python</groupId>
+ <artifactId>jython</artifactId>
+ <version>2.5.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr-runtime</artifactId>
+ <version>3.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>r09</version>
+ </dependency>
+
</dependencies>
</dependencyManagement>
Modified: incubator/oozie/branches/3.1/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/release-log.txt?rev=1228428&r1=1228427&r2=1228428&view=diff
==============================================================================
--- incubator/oozie/branches/3.1/release-log.txt (original)
+++ incubator/oozie/branches/3.1/release-log.txt Fri Jan 6 21:57:18 2012
@@ -1,3 +1,6 @@
+-- Oozie 3.1.1 release
+OOZIE-564 Embedded pig within python fails when run using oozie
+
-- Oozie 3.1.0 release
OOZIE-37 Documentation related modifications for "log -action" option
OOZIE-35 add auto-rerun for error codes JA008 and JA009 in action executor
Modified: incubator/oozie/branches/3.1/sharelib/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/branches/3.1/sharelib/pom.xml?rev=1228428&r1=1228427&r2=1228428&view=diff
==============================================================================
--- incubator/oozie/branches/3.1/sharelib/pom.xml (original)
+++ incubator/oozie/branches/3.1/sharelib/pom.xml Fri Jan 6 21:57:18 2012
@@ -72,6 +72,24 @@
<artifactId>jline</artifactId>
<scope>compile</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.python</groupId>
+ <artifactId>jython</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr-runtime</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <scope>runtime</scope>
+ </dependency>
</dependencies>
<build>