You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/09/02 08:15:50 UTC
svn commit: r1621928 - in /pig/trunk: ./
src/docs/src/documentation/content/xdocs/ test/org/apache/pig/pigunit/
test/org/apache/pig/test/pigunit/ test/org/apache/pig/test/pigunit/pig/
Author: daijy
Date: Tue Sep 2 06:15:50 2014
New Revision: 1621928
URL: http://svn.apache.org/r1621928
Log:
PIG-4144: Make pigunit.PigTest work in tez mode
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/docs/src/documentation/content/xdocs/test.xml
pig/trunk/test/org/apache/pig/pigunit/MiniClusterRunner.java
pig/trunk/test/org/apache/pig/pigunit/PigTest.java
pig/trunk/test/org/apache/pig/test/pigunit/TestPigTest.java
pig/trunk/test/org/apache/pig/test/pigunit/pig/TestGruntParser.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1621928&r1=1621927&r2=1621928&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Sep 2 06:15:50 2014
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-4144: Make pigunit.PigTest work in tez mode (daijy)
+
PIG-4128: New logical optimizer rule: ConstantCalculator (daijy)
PIG-4124: Command for Python streaming udf should be configurable (cheolsoo)
Modified: pig/trunk/src/docs/src/documentation/content/xdocs/test.xml
URL: http://svn.apache.org/viewvc/pig/trunk/src/docs/src/documentation/content/xdocs/test.xml?rev=1621928&r1=1621927&r2=1621928&view=diff
==============================================================================
--- pig/trunk/src/docs/src/documentation/content/xdocs/test.xml (original)
+++ pig/trunk/src/docs/src/documentation/content/xdocs/test.xml Tue Sep 2 06:15:50 2014
@@ -829,8 +829,8 @@ $pig_trunk ant pigunit-jar
<!-- +++++++++++++++++++++++++++++++++++++++ -->
<section>
<title>Mapreduce Mode</title>
- <p>PigUnit also runs in Pig's mapreduce mode. Mapreduce mode requires you to use a Hadoop cluster and HDFS installation.
- It is enabled when the Java system property pigunit.exectype.cluster is set to any value: e.g. -Dpigunit.exectype.cluster=true or System.getProperties().setProperty("pigunit.exectype.cluster", "true"). The cluster you select must be specified in the CLASSPATH (similar to the HADOOP_CONF_DIR variable).
+ <p>PigUnit also runs in Pig's mapreduce/tez/tez_local mode. Mapreduce/Tez mode requires you to use a Hadoop cluster and HDFS installation.
+ It is enabled when the Java system property pigunit.exectype is set to specific values (mr/tez/tez_local): e.g. -Dpigunit.exectype=mr or System.getProperties().setProperty("pigunit.exectype", "mr"), which means PigUnit will run in mr mode. The cluster you select to run mr/tez test must be specified in the CLASSPATH (similar to the HADOOP_CONF_DIR variable).
</p>
</section>
</section>
Modified: pig/trunk/test/org/apache/pig/pigunit/MiniClusterRunner.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/pigunit/MiniClusterRunner.java?rev=1621928&r1=1621927&r2=1621928&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/pigunit/MiniClusterRunner.java (original)
+++ pig/trunk/test/org/apache/pig/pigunit/MiniClusterRunner.java Tue Sep 2 06:15:50 2014
@@ -12,7 +12,7 @@
*/
package org.apache.pig.pigunit;
-import org.apache.pig.test.MiniCluster;
+import org.apache.pig.test.MiniGenericCluster;
/**
@@ -41,6 +41,6 @@ import org.apache.pig.test.MiniCluster;
public class MiniClusterRunner {
public static void main(String[] args) {
System.setProperty("hadoop.log.dir", "/tmp/pigunit");
- MiniCluster.buildCluster();
+ MiniGenericCluster.buildCluster();
}
}
Modified: pig/trunk/test/org/apache/pig/pigunit/PigTest.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/pigunit/PigTest.java?rev=1621928&r1=1621927&r2=1621928&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/pigunit/PigTest.java (original)
+++ pig/trunk/test/org/apache/pig/pigunit/PigTest.java Tue Sep 2 06:15:50 2014
@@ -31,6 +31,8 @@ import junit.framework.Assert;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.pig.ExecType;
+import org.apache.pig.ExecTypeProvider;
+import org.apache.pig.PigException;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
@@ -62,7 +64,7 @@ public class PigTest {
private static ThreadLocal<Cluster> cluster = new ThreadLocal<Cluster>();
private static final Logger LOG = Logger.getLogger(PigTest.class);
- private static final String EXEC_CLUSTER = "pigunit.exectype.cluster";
+ private static final String EXEC_CLUSTER = "pigunit.exectype";
/**
* Initializes the Pig test.
@@ -120,16 +122,30 @@ public class PigTest {
* @throws ExecException If the PigServer can't be started.
*/
public static Cluster getCluster() throws ExecException {
- if (cluster.get() == null) {
- if (System.getProperties().containsKey(EXEC_CLUSTER)) {
- LOG.info("Using cluster mode");
- pig.set(new PigServer(ExecType.MAPREDUCE));
- } else {
- LOG.info("Using default local mode");
- pig.set(new PigServer(ExecType.LOCAL));
+ try {
+ if (cluster.get() == null) {
+ ExecType execType = ExecType.LOCAL;
+ if (System.getProperties().containsKey(EXEC_CLUSTER)) {
+ if (System.getProperties().getProperty(EXEC_CLUSTER).equalsIgnoreCase("mr")) {
+ LOG.info("Using mr cluster mode");
+ execType = ExecType.MAPREDUCE;
+ } else if (System.getProperties().getProperty(EXEC_CLUSTER).equalsIgnoreCase("tez")) {
+ LOG.info("Using tez cluster mode");
+ execType = ExecTypeProvider.fromString("tez");
+ } else if (System.getProperties().getProperty(EXEC_CLUSTER).equalsIgnoreCase("tez_local")) {
+ LOG.info("Using tez local mode");
+ execType = ExecTypeProvider.fromString("tez_local");
+ } else {
+ LOG.info("Using default local mode");
+ }
+ } else {
+ LOG.info("Using default local mode");
+ }
+ pig.set(new PigServer(execType));
+ cluster.set(new Cluster(pig.get().getPigContext()));
}
-
- cluster.set(new Cluster(pig.get().getPigContext()));
+ } catch (PigException e) {
+ throw new ExecException(e);
}
return cluster.get();
Modified: pig/trunk/test/org/apache/pig/test/pigunit/TestPigTest.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/pigunit/TestPigTest.java?rev=1621928&r1=1621927&r2=1621928&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/pigunit/TestPigTest.java (original)
+++ pig/trunk/test/org/apache/pig/test/pigunit/TestPigTest.java Tue Sep 2 06:15:50 2014
@@ -27,10 +27,10 @@ import org.apache.commons.lang.StringUti
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
-import org.apache.pig.ExecType;
import org.apache.pig.pigunit.Cluster;
import org.apache.pig.pigunit.PigTest;
import org.apache.pig.pigunit.pig.PigServer;
+import org.apache.pig.test.Util;
import org.apache.pig.tools.parameters.ParseException;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -56,7 +56,8 @@ public class TestPigTest {
private static final Log LOG = LogFactory.getLog(TestPigTest.class);
@BeforeClass
- public static void setUpOnce() throws IOException {
+ public static void setUpOnce() throws Exception {
+ System.getProperties().setProperty("pigunit.exectype", Util.getLocalTestMode().toString());
cluster = PigTest.getCluster();
cluster.update(
@@ -366,12 +367,11 @@ public class TestPigTest {
/**
* This is a test for default bootup. PIG-2456
- *
- * @throws IOException
+ * @throws Exception
*/
@Test
- public void testDefaultBootup() throws ParseException, IOException {
+ public void testDefaultBootup() throws Exception {
// Test with properties file
String pigProps = "pig.properties";
String bootupPath = "/tmp/.temppigbootup";
@@ -414,13 +414,7 @@ public class TestPigTest {
// Create a pigunit.pig.PigServer and Cluster to run this test.
PigServer pig = null;
- if (System.getProperties().containsKey("pigunit.exectype.cluster")) {
- LOG.info("Using cluster mode");
- pig = new PigServer(ExecType.MAPREDUCE);
- } else {
- LOG.info("Using default local mode");
- pig = new PigServer(ExecType.LOCAL);
- }
+ pig = new PigServer(Util.getLocalTestMode());
final Cluster cluster = new Cluster(pig.getPigContext());
Modified: pig/trunk/test/org/apache/pig/test/pigunit/pig/TestGruntParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/pigunit/pig/TestGruntParser.java?rev=1621928&r1=1621927&r2=1621928&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/pigunit/pig/TestGruntParser.java (original)
+++ pig/trunk/test/org/apache/pig/test/pigunit/pig/TestGruntParser.java Tue Sep 2 06:15:50 2014
@@ -19,10 +19,10 @@ import java.util.Map;
import junit.framework.Assert;
-import org.apache.pig.ExecType;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.pigunit.pig.GruntParser;
import org.apache.pig.pigunit.pig.PigServer;
+import org.apache.pig.test.Util;
import org.junit.Before;
import org.junit.Test;
@@ -33,12 +33,12 @@ public class TestGruntParser {
@SuppressWarnings("serial")
@Before
- public void setUp() throws ExecException {
+ public void setUp() throws Exception {
override = new HashMap<String, String>() {{
put("STORE", "");
put("DUMP", "");
}};
- PigServer pigServer = new PigServer(ExecType.LOCAL);
+ PigServer pigServer = new PigServer(Util.getLocalTestMode());
parser = new GruntParser(new StringReader(""), pigServer, override);
}