You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by xu...@apache.org on 2016/07/08 02:38:18 UTC
svn commit: r1751849 - in /pig/branches/spark:
src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
test/org/apache/pig/test/TestEvalPipelineLocal.java
Author: xuefu
Date: Fri Jul 8 02:38:18 2016
New Revision: 1751849
URL: http://svn.apache.org/viewvc?rev=1751849&view=rev
Log:
PIG-4944: Reset UDFContext#jobConf in spark mode (Liyun via Xuefu)
Modified:
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
pig/branches/spark/test/org/apache/pig/test/TestEvalPipelineLocal.java
Modified: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1751849&r1=1751848&r2=1751849&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java (original)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java Fri Jul 8 02:38:18 2016
@@ -224,9 +224,14 @@ public class SparkLauncher extends Launc
new JobGraphBuilder(sparkplan, convertMap, sparkStats, sparkContext, jobMetricsListener, jobGroupID, jobConf, pigContext).visit();
cleanUpSparkJob(sparkStats);
sparkStats.finish();
+ resetUDFContext();
return sparkStats;
}
+ private void resetUDFContext() {
+ UDFContext.getUDFContext().addJobConf(null);
+ }
+
private void uploadResources(SparkOperPlan sparkPlan) throws IOException {
addFilesToSparkJob();
addJarsToSparkJob(sparkPlan);
Modified: pig/branches/spark/test/org/apache/pig/test/TestEvalPipelineLocal.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestEvalPipelineLocal.java?rev=1751849&r1=1751848&r2=1751849&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestEvalPipelineLocal.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestEvalPipelineLocal.java Fri Jul 8 02:38:18 2016
@@ -65,7 +65,7 @@ import org.junit.Before;
import org.junit.Test;
public class TestEvalPipelineLocal {
-
+
private PigServer pigServer;
static final int MAX_SIZE = 100000;
@@ -1114,8 +1114,6 @@ public class TestEvalPipelineLocal {
@Test
public void testSetLocationCalledInFE() throws Exception {
- // Need to reset it when running multiple testcases
- UDFContext.getUDFContext().addJobConf(null);
File f1 = createFile(new String[]{"a","b"});
pigServer.registerQuery("a = load '" + Util.generateURI(f1.toString(), pigServer.getPigContext())
+ "' using " + SetLocationTestLoadFunc.class.getName()