You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by xu...@apache.org on 2016/07/05 03:46:51 UTC
svn commit: r1751389 - in /pig/branches/spark:
src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
test/org/apache/pig/test/TestFinish.java
Author: xuefu
Date: Tue Jul 5 03:46:51 2016
New Revision: 1751389
URL: http://svn.apache.org/viewvc?rev=1751389&view=rev
Log:
PIG-4281: Fix TestFinish for Spark engine (Liyun via Xuefu)
Modified:
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
pig/branches/spark/test/org/apache/pig/test/TestFinish.java
Modified: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1751389&r1=1751388&r2=1751389&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java (original)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java Tue Jul 5 03:46:51 2016
@@ -47,6 +47,7 @@ import org.apache.pig.backend.BackendExc
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.backend.hadoop.executionengine.Launcher;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCollectedGroup;
@@ -650,5 +651,6 @@ public class SparkLauncher extends Launc
SchemaTupleBackend.initialize(jobConf, pigContext);
Utils.setDefaultTimeZone(jobConf);
+ PigMapReduce.sJobConfInternal.set(jobConf);
}
}
Modified: pig/branches/spark/test/org/apache/pig/test/TestFinish.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestFinish.java?rev=1751389&r1=1751388&r2=1751389&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestFinish.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestFinish.java Tue Jul 5 03:46:51 2016
@@ -45,7 +45,7 @@ public class TestFinish {
BagFactory mBf = BagFactory.getInstance();
File f1;
- static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
+ private static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
static public class MyEvalFunction extends EvalFunc<Tuple> {
String execType;
@@ -67,12 +67,7 @@ public class TestFinish {
@Override
public void finish() {
try {
- FileSystem fs = null;
- if (execType.equalsIgnoreCase("SPARK")) {
- fs = FileSystem.get(cluster.getConfiguration());
- } else {
- fs = FileSystem.get(PigMapReduce.sJobConfInternal.get());
- }
+ FileSystem fs = FileSystem.get(PigMapReduce.sJobConfInternal.get());
fs.create(new Path(expectedFileName));
} catch (IOException e) {
throw new RuntimeException("Unable to create file:" + expectedFileName);
@@ -141,7 +136,7 @@ public class TestFinish {
String inputFileName = setUp(cluster.getExecType());
// this file will be created on the cluster if finish() is called
String expectedFileName = "testFinishInMapMR-finish.txt";
- pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('"+cluster.getExecType()+"','"
+ pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('MAPREDUCE','"
+ expectedFileName + "');");
pigServer.registerQuery("a = load '" + Util.encodeEscape(inputFileName) + "' using "
+ PigStorage.class.getName() + "(':');");
@@ -160,7 +155,7 @@ public class TestFinish {
String inputFileName = setUp(cluster.getExecType());
// this file will be created on the cluster if finish() is called
String expectedFileName = "testFinishInReduceMR-finish.txt";
- pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('"+cluster.getExecType()+"','"
+ pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('MAPREDUCE','"
+ expectedFileName + "');");
pigServer.registerQuery("a = load '" + Util.encodeEscape(inputFileName) + "' using "
+ PigStorage.class.getName() + "(':');");