You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2016/01/05 09:21:05 UTC
svn commit: r1723015 - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/
test/org/apache/pig/test/
Author: rohini
Date: Tue Jan 5 08:21:04 2016
New Revision: 1723015
URL: http://svn.apache.org/viewvc?rev=1723015&view=rev
Log:
PIG-4768: EvalFunc reporter is null in Tez (rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ProgressableReporter.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1723015&r1=1723014&r2=1723015&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jan 5 08:21:04 2016
@@ -77,6 +77,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4768: EvalFunc reporter is null in Tez (rohini)
+
PIG-4760: TezDAGStats.convertToHadoopCounters is not used, but impose MR counter limit (daijy)
PIG-4755: Typo in runpigmix script (mitdesai via daijy)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ProgressableReporter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ProgressableReporter.java?rev=1723015&r1=1723014&r2=1723015&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ProgressableReporter.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ProgressableReporter.java Tue Jan 5 08:21:04 2016
@@ -17,8 +17,6 @@
*/
package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer;
-import java.io.IOException;
-
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PigProgressable;
@@ -26,7 +24,7 @@ public class ProgressableReporter implem
TaskAttemptContext rep;
public ProgressableReporter(){
-
+
}
public ProgressableReporter(TaskAttemptContext rep) {
@@ -34,14 +32,19 @@ public class ProgressableReporter implem
this.rep = rep;
}
+ @Override
public void progress() {
- if(rep!=null)
+ if (rep != null) {
rep.progress();
+ }
}
+ @Override
public void progress(String msg) {
try {
- rep.setStatus(msg);
+ if (rep != null) {
+ rep.setStatus(msg);
+ }
}catch (Exception e) {
rep.progress();
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java?rev=1723015&r1=1723014&r2=1723015&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java Tue Jan 5 08:21:04 2016
@@ -38,6 +38,7 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRConfiguration;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigHadoopLogger;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.ProgressableReporter;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.UDFFinishVisitor;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
@@ -117,6 +118,11 @@ public class PigProcessor extends Abstra
// Reset static variables cleared for avoiding OOM.
new JVMReuseImpl().cleanupStaticData();
+ // Set an empty reporter for now. Once we go to Tez 0.8
+ // which adds support for mapreduce like progress (TEZ-808),
+ // we need to call progress on Tez API
+ PhysicalOperator.setReporter(new ProgressableReporter());
+
UserPayload payload = getContext().getUserPayload();
conf = TezUtils.createConfFromUserPayload(payload);
PigContext.setPackageImportList((ArrayList<String>) ObjectSerializer
Modified: pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java?rev=1723015&r1=1723014&r2=1723015&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java Tue Jan 5 08:21:04 2016
@@ -249,6 +249,7 @@ public class TestEvalPipeline {
return sb.toString();
}
+ @Override
public Schema outputSchema(Schema input) {
try {
Schema stringSchema = new Schema(new Schema.FieldSchema(null, DataType.CHARARRAY));
@@ -264,6 +265,8 @@ public class TestEvalPipeline {
@Override
public Map<String, Object> exec(Tuple input) throws IOException {
+ super.reporter.progress();
+
TupleFactory tupleFactory = TupleFactory.getInstance();
ArrayList<Object> objList = new ArrayList<Object>();
objList.add(new Integer(1));
@@ -293,6 +296,7 @@ public class TestEvalPipeline {
return myMap;
}
+ @Override
public Schema outputSchema(Schema input) {
return new Schema(new Schema.FieldSchema(null, DataType.MAP));
}