You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by rd...@apache.org on 2011/05/19 21:42:35 UTC
svn commit: r1125072 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/tools/pigstats/JobStats.java
src/org/apache/pig/tools/pigstats/PigStatsUtil.java
Author: rding
Date: Thu May 19 19:42:35 2011
New Revision: 1125072
URL: http://svn.apache.org/viewvc?rev=1125072&view=rev
Log:
PIG-2029: Inconsistency in Pig Stats reports
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java
pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1125072&r1=1125071&r2=1125072&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu May 19 19:42:35 2011
@@ -227,6 +227,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-2029: Inconsistency in Pig Stats reports (rding)
+
PIG-2070: "Unknown" appears in error message for an error case (thejas)
PIG-2069: LoadFunc jar does not ship to backend in MultiQuery case (rding)
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java?rev=1125072&r1=1125071&r2=1125072&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java Thu May 19 19:42:35 2011
@@ -302,14 +302,22 @@ public final class JobStats extends Oper
} else if (state == JobState.SUCCESS) {
sb.append(id).append("\t")
.append(numberMaps).append("\t")
- .append(numberReduces).append("\t")
- .append(maxMapTime/1000).append("\t")
- .append(minMapTime/1000).append("\t")
- .append(avgMapTime/1000).append("\t")
- .append(maxReduceTime/1000).append("\t")
- .append(minReduceTime/1000).append("\t")
- .append(avgReduceTime/1000).append("\t")
- .append(getAlias()).append("\t")
+ .append(numberReduces).append("\t");
+ if (maxMapTime < 0) {
+ sb.append("n/a\t").append("n/a\t").append("n/a\t");
+ } else {
+ sb.append(maxMapTime/1000).append("\t")
+ .append(minMapTime/1000).append("\t")
+ .append(avgMapTime/1000).append("\t");
+ }
+ if (maxReduceTime < 0) {
+ sb.append("n/a\t").append("n/a\t").append("n/a\t");
+ } else {
+ sb.append(maxReduceTime/1000).append("\t")
+ .append(minReduceTime/1000).append("\t")
+ .append(avgReduceTime/1000).append("\t");
+ }
+ sb.append(getAlias()).append("\t")
.append(getFeature()).append("\t");
}
for (OutputStats os : outputs) {
@@ -374,7 +382,7 @@ public final class JobStats extends Oper
}
}
- void addMapReduceStatistics(JobClient client) {
+ void addMapReduceStatistics(JobClient client, Configuration conf) {
TaskReport[] maps = null;
try {
maps = client.getMapTaskReports(jobId);
@@ -394,7 +402,13 @@ public final class JobStats extends Oper
}
long avg = total / size;
setMapStat(size, max, min, avg);
+ } else {
+ int m = conf.getInt("mapred.map.tasks", 1);
+ if (m > 0) {
+ setMapStat(m, -1, -1, -1);
+ }
}
+
TaskReport[] reduces = null;
try {
reduces = client.getReduceTaskReports(jobId);
@@ -414,7 +428,12 @@ public final class JobStats extends Oper
}
long avg = total / size;
setReduceStat(size, max, min, avg);
- }
+ } else {
+ int m = conf.getInt("mapred.reduce.tasks", 1);
+ if (m > 0) {
+ setReduceStat(m, -1, -1, -1);
+ }
+ }
}
void setAlias(MapReduceOper mro) {
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java?rev=1125072&r1=1125071&r2=1125072&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java Thu May 19 19:42:35 2011
@@ -324,7 +324,7 @@ public abstract class PigStatsUtil {
} else {
js.setSuccessful(true);
- js.addMapReduceStatistics(ps.getJobClient());
+ js.addMapReduceStatistics(ps.getJobClient(), job.getJobConf());
JobClient client = ps.getJobClient();
RunningJob rjob = null;