You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2014/09/04 23:39:20 UTC
svn commit: r1622567 - in /pig/branches/spark: ./ src/
test/org/apache/pig/test/
test/org/apache/pig/test/data/bzipdir1.bz2/bzipdir2.bz2/
test/org/apache/pig/tez/
Author: cheolsoo
Date: Thu Sep 4 21:39:19 2014
New Revision: 1622567
URL: http://svn.apache.org/r1622567
Log:
Merge latest trunk changes
Modified:
pig/branches/spark/ (props changed)
pig/branches/spark/src/pig-default.properties (props changed)
pig/branches/spark/test/org/apache/pig/test/TestGroupConstParallel.java
pig/branches/spark/test/org/apache/pig/test/TestGroupConstParallelMR.java
pig/branches/spark/test/org/apache/pig/test/TestJobSubmission.java
pig/branches/spark/test/org/apache/pig/test/TestJobSubmissionMR.java
pig/branches/spark/test/org/apache/pig/test/data/bzipdir1.bz2/bzipdir2.bz2/recordLossblockHeaderEndsAt136500.txt.bz2 (props changed)
pig/branches/spark/test/org/apache/pig/tez/TestGroupConstParallelTez.java
pig/branches/spark/test/org/apache/pig/tez/TestJobSubmissionTez.java
Propchange: pig/branches/spark/
------------------------------------------------------------------------------
Merged /pig/trunk:r1622521-1622566
Propchange: pig/branches/spark/src/pig-default.properties
------------------------------------------------------------------------------
Merged /pig/trunk/src/pig-default.properties:r1622521-1622566
Modified: pig/branches/spark/test/org/apache/pig/test/TestGroupConstParallel.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestGroupConstParallel.java?rev=1622567&r1=1622566&r2=1622567&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestGroupConstParallel.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestGroupConstParallel.java Thu Sep 4 21:39:19 2014
@@ -34,7 +34,6 @@ import org.apache.pig.newplan.optimizer.
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.PigStats.JobGraph;
import org.junit.AfterClass;
-import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@@ -42,11 +41,10 @@ import org.junit.Test;
public abstract class TestGroupConstParallel {
private static final String INPUT_FILE = "TestGroupConstParallelInp";
- private static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
+ private static MiniGenericCluster cluster = null;
-
- @BeforeClass
public static void oneTimeSetup() throws Exception{
+ cluster = MiniGenericCluster.buildCluster();
String[] input = {
"two",
"one",
@@ -58,7 +56,9 @@ public abstract class TestGroupConstPara
@AfterClass
public static void tearDownAfterClass() throws Exception {
- cluster.shutDown();
+ if (cluster!=null) {
+ cluster.shutDown();
+ }
}
/**
Modified: pig/branches/spark/test/org/apache/pig/test/TestGroupConstParallelMR.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestGroupConstParallelMR.java?rev=1622567&r1=1622566&r2=1622567&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestGroupConstParallelMR.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestGroupConstParallelMR.java Thu Sep 4 21:39:19 2014
@@ -31,8 +31,18 @@ import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.util.ConfigurationValidator;
import org.apache.pig.tools.pigstats.PigStats.JobGraph;
import org.apache.pig.tools.pigstats.mapreduce.MRJobStats;
+import org.junit.Assume;
+import org.junit.BeforeClass;
public class TestGroupConstParallelMR extends TestGroupConstParallel {
+
+ @BeforeClass
+ public static void oneTimeSetup() throws Exception{
+ String execType = System.getProperty("test.exec.type");
+ Assume.assumeTrue("This test suite should only run in mr mode", execType.equalsIgnoreCase("mr"));
+ TestGroupConstParallel.oneTimeSetup();
+ }
+
@Override
public void checkGroupAllWithParallelGraphResult(JobGraph jGraph) {
assertEquals(1, jGraph.size());
Modified: pig/branches/spark/test/org/apache/pig/test/TestJobSubmission.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestJobSubmission.java?rev=1622567&r1=1622566&r2=1622567&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestJobSubmission.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestJobSubmission.java Thu Sep 4 21:39:19 2014
@@ -53,7 +53,6 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assume;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@@ -72,10 +71,10 @@ abstract public class TestJobSubmission
String curDir;
String inpDir;
String golDir;
- static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
+ static MiniGenericCluster cluster = null;
- @BeforeClass
- public static void onetimeSetUp() throws Exception {
+ public static void oneTimeSetUp() throws Exception {
+ cluster = MiniGenericCluster.buildCluster();
pc = new PigContext(cluster.getExecType(), cluster.getProperties());
try {
pc.connect();
@@ -85,9 +84,6 @@ abstract public class TestJobSubmission
}
GenPhyOp.setPc(pc);
Util.copyFromLocalToCluster(cluster, "test/org/apache/pig/test/data/passwd", "/passwd");
-
- Configuration conf = cluster.getConfiguration();
-
}
@Before
@@ -107,7 +103,9 @@ abstract public class TestJobSubmission
@AfterClass
public static void oneTimeTearDown() throws Exception {
- cluster.shutDown();
+ if (cluster!=null) {
+ cluster.shutDown();
+ }
}
@Test
Modified: pig/branches/spark/test/org/apache/pig/test/TestJobSubmissionMR.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestJobSubmissionMR.java?rev=1622567&r1=1622566&r2=1622567&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestJobSubmissionMR.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestJobSubmissionMR.java Thu Sep 4 21:39:19 2014
@@ -30,8 +30,18 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.util.ConfigurationValidator;
+import org.junit.Assume;
+import org.junit.BeforeClass;
public class TestJobSubmissionMR extends TestJobSubmission {
+
+ @BeforeClass
+ public static void oneTimeSetup() throws Exception{
+ String execType = System.getProperty("test.exec.type");
+ Assume.assumeTrue("This test suite should only run in mr mode", execType.equalsIgnoreCase("mr"));
+ TestJobSubmission.oneTimeSetUp();
+ }
+
@Override
public void checkJobControlCompilerErrResult(PhysicalPlan pp, PigContext pc) throws Exception {
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
Propchange: pig/branches/spark/test/org/apache/pig/test/data/bzipdir1.bz2/bzipdir2.bz2/recordLossblockHeaderEndsAt136500.txt.bz2
------------------------------------------------------------------------------
Merged /pig/trunk/test/org/apache/pig/test/data/bzipdir1.bz2/bzipdir2.bz2/recordLossblockHeaderEndsAt136500.txt.bz2:r1622521-1622566
Modified: pig/branches/spark/test/org/apache/pig/tez/TestGroupConstParallelTez.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/tez/TestGroupConstParallelTez.java?rev=1622567&r1=1622566&r2=1622567&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/tez/TestGroupConstParallelTez.java (original)
+++ pig/branches/spark/test/org/apache/pig/tez/TestGroupConstParallelTez.java Thu Sep 4 21:39:19 2014
@@ -32,9 +32,18 @@ import org.apache.pig.tools.pigstats.Pig
import org.apache.pig.tools.pigstats.tez.TezTaskStats;
import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.Vertex;
+import org.junit.Assume;
+import org.junit.BeforeClass;
public class TestGroupConstParallelTez extends TestGroupConstParallel {
+ @BeforeClass
+ public static void oneTimeSetup() throws Exception{
+ String execType = System.getProperty("test.exec.type");
+ Assume.assumeTrue("This test suite should only run in tez mode", execType.equalsIgnoreCase("tez"));
+ TestGroupConstParallel.oneTimeSetup();
+ }
+
@Override
public void checkGroupAllWithParallelGraphResult(JobGraph jGraph) {
TezTaskStats ts = (TezTaskStats)jGraph.getSinks().get(0);
@@ -51,7 +60,7 @@ public class TestGroupConstParallelTez e
ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc);
parallelismSetter.visit();
- DAG tezDag = new DAG("test");
+ DAG tezDag = DAG.create("test");
TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null);
dagBuilder.visit();
for (Vertex v : tezDag.getVertices()) {
@@ -71,7 +80,7 @@ public class TestGroupConstParallelTez e
ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc);
parallelismSetter.visit();
- DAG tezDag = new DAG("test");
+ DAG tezDag = DAG.create("test");
TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null);
dagBuilder.visit();
for (Vertex v : tezDag.getVertices()) {
Modified: pig/branches/spark/test/org/apache/pig/tez/TestJobSubmissionTez.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/tez/TestJobSubmissionTez.java?rev=1622567&r1=1622566&r2=1622567&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/tez/TestJobSubmissionTez.java (original)
+++ pig/branches/spark/test/org/apache/pig/tez/TestJobSubmissionTez.java Thu Sep 4 21:39:19 2014
@@ -35,8 +35,18 @@ import org.apache.pig.test.Util;
import org.apache.tez.common.TezUtils;
import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.Vertex;
+import org.junit.Assume;
+import org.junit.BeforeClass;
public class TestJobSubmissionTez extends TestJobSubmission {
+
+ @BeforeClass
+ public static void oneTimeSetup() throws Exception{
+ String execType = System.getProperty("test.exec.type");
+ Assume.assumeTrue("This test suite should only run in tez mode", execType.equalsIgnoreCase("tez"));
+ TestJobSubmission.oneTimeSetUp();
+ }
+
@Override
public void checkJobControlCompilerErrResult(PhysicalPlan pp, PigContext pc) throws Exception {
TezOperPlan tezPlan = buildTezPlan(pp, pc);
@@ -47,7 +57,7 @@ public class TestJobSubmissionTez extend
ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc);
parallelismSetter.visit();
- DAG tezDag = new DAG("test");
+ DAG tezDag = DAG.create("test");
TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null);
try {
dagBuilder.visit();
@@ -66,7 +76,7 @@ public class TestJobSubmissionTez extend
ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc);
parallelismSetter.visit();
- DAG tezDag = new DAG("test");
+ DAG tezDag = DAG.create("test");
TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null);
dagBuilder.visit();
for (Vertex v : tezDag.getVertices()) {