You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2013/06/03 16:49:52 UTC
svn commit: r1489014 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
Author: rohini
Date: Mon Jun 3 14:49:52 2013
New Revision: 1489014
URL: http://svn.apache.org/r1489014
Log:
PIG-3327: Pig hits OOM when fetching task reports (rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1489014&r1=1489013&r2=1489014&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Jun 3 14:49:52 2013
@@ -28,6 +28,8 @@ PIG-3174: Remove rpm and deb artifacts
IMPROVEMENTS
+PIG-3327: Pig hits OOM when fetching task reports (rohini)
+
PIG-3336: Change IN operator to use or-expressions instead of EvalFunc (cheolsoo)
PIG-3339: Move pattern compilation in ToDate as a static variable (rohini)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java?rev=1489014&r1=1489013&r2=1489014&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/Launcher.java Mon Jun 3 14:49:52 2013
@@ -148,9 +148,11 @@ public abstract class Launcher {
TaskReport[] mapRep = jobClient.getMapTaskReports(MRJobID);
getErrorMessages(mapRep, "map", errNotDbg, pigContext);
totalHadoopTimeSpent += computeTimeSpent(mapRep);
+ mapRep = null;
TaskReport[] redRep = jobClient.getReduceTaskReports(MRJobID);
getErrorMessages(redRep, "reduce", errNotDbg, pigContext);
- totalHadoopTimeSpent += computeTimeSpent(mapRep);
+ totalHadoopTimeSpent += computeTimeSpent(redRep);
+ redRep = null;
} catch (IOException e) {
if(job.getState() == Job.SUCCESS) {
// if the job succeeded, let the user know that
@@ -162,9 +164,9 @@ public abstract class Launcher {
}
}
- protected long computeTimeSpent(TaskReport[] mapReports) {
+ protected long computeTimeSpent(TaskReport[] taskReports) {
long timeSpent = 0;
- for (TaskReport r : mapReports) {
+ for (TaskReport r : taskReports) {
timeSpent += (r.getFinishTime() - r.getStartTime());
}
return timeSpent;