You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2014/08/21 18:59:17 UTC
svn commit: r1619485 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainer.java
src/org/apache/pig/impl/builtin/StreamingUDF.java
test/org/apache/pig/impl/builtin/TestStreamingUDF.java test/tez-tests
Author: cheolsoo
Date: Thu Aug 21 16:59:17 2014
New Revision: 1619485
URL: http://svn.apache.org/r1619485
Log:
PIG-4061: Make Streaming UDF work in Tez (hotfix PIG-4061-3.patch)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainer.java
pig/trunk/src/org/apache/pig/impl/builtin/StreamingUDF.java
pig/trunk/test/org/apache/pig/impl/builtin/TestStreamingUDF.java
pig/trunk/test/tez-tests
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1619485&r1=1619484&r2=1619485&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Aug 21 16:59:17 2014
@@ -64,6 +64,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4061: Make Streaming UDF work in Tez (hotfix PIG-4061-3.patch)
+
PIG-4134: TEZ-1449 broke the build (knoguchi)
PIG-4132: TEZ-1246 and TEZ-1390 broke a build (knoguchi)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainer.java?rev=1619485&r1=1619484&r2=1619485&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainer.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainer.java Thu Aug 21 16:59:17 2014
@@ -30,7 +30,6 @@ import java.util.Set;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.builtin.StreamingUDF;
import org.apache.pig.impl.plan.NodeIdGenerator;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.OperatorPlan;
@@ -92,11 +91,6 @@ public class TezPlanContainer extends Op
}
URI jarUri = new File(jarName).toURI();
jarLists.add(jarUri);
- if ("StreamingUDF".equals(clazz.getSimpleName())) {
- for (String fileName : StreamingUDF.getResourcesForJar()) {
- jarLists.add(new File(fileName).toURI());
- }
- }
}
}
Modified: pig/trunk/src/org/apache/pig/impl/builtin/StreamingUDF.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/builtin/StreamingUDF.java?rev=1619485&r1=1619484&r2=1619485&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/builtin/StreamingUDF.java (original)
+++ pig/trunk/src/org/apache/pig/impl/builtin/StreamingUDF.java Thu Aug 21 16:59:17 2014
@@ -26,8 +26,6 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
@@ -344,21 +342,6 @@ public class StreamingUDF extends EvalFu
return language.toLowerCase().startsWith("python");
}
- /**
- * Returns a list of file names (relative to root of pig jar) of files that need to be
- * included in the jar shipped to the cluster.
- *
- * Will need to be smarter as more languages are added and the controller files are large.
- *
- * @return
- */
- public static List<String> getResourcesForJar() {
- List<String> files = new ArrayList<String>();
- files.add(PYTHON_CONTROLLER_JAR_PATH);
- files.add(PYTHON_PIG_UTIL_PATH);
- return files;
- }
-
private Object getOutput(Tuple input) throws ExecException {
if (outputQueue == null) {
throw new ExecException("Process has already been shut down. No way to retrieve output for input: " + input);
Modified: pig/trunk/test/org/apache/pig/impl/builtin/TestStreamingUDF.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/impl/builtin/TestStreamingUDF.java?rev=1619485&r1=1619484&r2=1619485&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/impl/builtin/TestStreamingUDF.java (original)
+++ pig/trunk/test/org/apache/pig/impl/builtin/TestStreamingUDF.java Thu Aug 21 16:59:17 2014
@@ -32,21 +32,27 @@ import org.apache.pig.builtin.mock.Stora
import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
-import org.apache.pig.test.MiniCluster;
+import org.apache.pig.test.MiniGenericCluster;
import org.apache.pig.test.Util;
import org.joda.time.DateTime;
+import org.junit.Before;
import org.junit.Test;
public class TestStreamingUDF {
private static PigServer pigServerLocal = null;
private static PigServer pigServerMapReduce = null;
-
+
private TupleFactory tf = TupleFactory.getInstance();
- private static MiniCluster cluster = MiniCluster.buildCluster();
-
+ private static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
+
+ @Before
+ public void setUp() throws Exception {
+ Util.resetStateForExecModeSwitch();
+ }
+
@Test
public void testPythonUDF_onCluster() throws Exception {
- pigServerMapReduce = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ pigServerMapReduce = new PigServer(cluster.getExecType(), cluster.getProperties());
String[] pythonScript = {
"from pig_util import outputSchema",
Modified: pig/trunk/test/tez-tests
URL: http://svn.apache.org/viewvc/pig/trunk/test/tez-tests?rev=1619485&r1=1619484&r2=1619485&view=diff
==============================================================================
--- pig/trunk/test/tez-tests (original)
+++ pig/trunk/test/tez-tests Thu Aug 21 16:59:17 2014
@@ -52,6 +52,7 @@
**/TestStoreInstances.java
**/TestStoreOld.java
**/TestStreaming.java
+**/TestStreamingUDF.java
**/TestToolsPigServer.java
**/TestUDF.java
**/TestUDFContext.java