You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/08/13 08:07:32 UTC
svn commit: r1617664 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainer.java
src/org/apache/pig/scripting/ScriptingOutputCapturer.java
Author: daijy
Date: Wed Aug 13 06:07:32 2014
New Revision: 1617664
URL: http://svn.apache.org/r1617664
Log:
PIG-4061: Make Streaming UDF work in Tez
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainer.java
pig/trunk/src/org/apache/pig/scripting/ScriptingOutputCapturer.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1617664&r1=1617663&r2=1617664&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Aug 13 06:07:32 2014
@@ -58,6 +58,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4061: Make Streaming UDF work in Tez (daijy)
+
PIG-4118: Fix hadoopversion 23 compilation due to TEZ-1237/TEZ-1407 (daijy)
PIG-4109: register local jar fail on Windows when Pig script is remote (daijy)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainer.java?rev=1617664&r1=1617663&r2=1617664&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainer.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPlanContainer.java Wed Aug 13 06:07:32 2014
@@ -30,6 +30,7 @@ import java.util.Set;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.builtin.StreamingUDF;
import org.apache.pig.impl.plan.NodeIdGenerator;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.OperatorPlan;
@@ -91,6 +92,11 @@ public class TezPlanContainer extends Op
}
URI jarUri = new File(jarName).toURI();
jarLists.add(jarUri);
+ if ("StreamingUDF".equals(clazz.getSimpleName())) {
+ for (String fileName : StreamingUDF.getResourcesForJar()) {
+ jarLists.add(new File(fileName).toURI());
+ }
+ }
}
}
@@ -99,14 +105,6 @@ public class TezPlanContainer extends Op
jarLists.add(scriptUDFJarFile.toURI());
}
- // Streaming UDF's are not working under Hadoop 2 (PIG-3478), so don't bother adding
- // resources for them yet.
- // if ("StreamingUDF".equals(clazz.getSimpleName())) {
- // for (String fileName : StreamingUDF.getResourcesForJar()) {
- // jarLists.add(new File(fileName).toURI().toURL());
- // }
- // }
-
return TezResourceManager.getInstance().addTezResources(jarLists);
}
Modified: pig/trunk/src/org/apache/pig/scripting/ScriptingOutputCapturer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/ScriptingOutputCapturer.java?rev=1617664&r1=1617663&r2=1617664&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/ScriptingOutputCapturer.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/ScriptingOutputCapturer.java Wed Aug 13 06:07:32 2014
@@ -66,7 +66,13 @@ public class ScriptingOutputCapturer {
String jobId = conf.get(MRConfiguration.JOB_ID);
String taskId = conf.get(MRConfiguration.TASK_ID);
- String hadoopLogDir = System.getProperty("hadoop.log.dir");
+ String hadoopLogDir = System.getProperty("yarn.app.container.log.dir");
+ if (hadoopLogDir == null) {
+ hadoopLogDir = conf.get("yarn.app.container.log.dir");
+ }
+ if (hadoopLogDir == null) {
+ hadoopLogDir = System.getProperty("hadoop.log.dir");
+ }
if (hadoopLogDir == null) {
hadoopLogDir = conf.get("hadoop.log.dir");
}