You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by rd...@apache.org on 2010/03/18 23:49:12 UTC
svn commit: r925034 - in /hadoop/pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
src/org/apache/pig/tools/pigstats/PigStats.java
Author: rding
Date: Thu Mar 18 22:49:12 2010
New Revision: 925034
URL: http://svn.apache.org/viewvc?rev=925034&view=rev
Log:
PIG-1266: Show spill count on the pig console at the end of the job
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=925034&r1=925033&r2=925034&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Thu Mar 18 22:49:12 2010
@@ -157,6 +157,9 @@ OPTIMIZATIONS
BUG FIXES
+PIG-1266: Show spill count on the pig console at the end of the job (sriranjan
+via rding)
+
PIG-1296: Skewed join fail due to negative partition index (daijy)
PIG-1293: pig wrapper script tends to fail if pig is in the path and PIG_HOME
Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=925034&r1=925033&r2=925034&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java Thu Mar 18 22:49:12 2010
@@ -336,6 +336,16 @@ public class MapReduceLauncher extends L
} else {
log.info("Bytes written : " + stats.getBytesWritten());
}
+ if(stats.getSMMSpillCount()==-1) {
+ log.info("Spillable Memory Manager spill count : Unable to determine spillable memory manager spill count");
+ } else {
+ log.info("Spillable Memory Manager spill count : " + stats.getSMMSpillCount());
+ }
+ if(stats.getProactiveSpillCount() == -1) {
+ log.info("Proactive spill count : Unable to determine proactive spill count");
+ } else {
+ log.info("Proactive spill count : " + stats.getProactiveSpillCount());
+ }
}
if (!failed) {
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java?rev=925034&r1=925033&r2=925034&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java Thu Mar 18 22:49:12 2010
@@ -316,6 +316,21 @@ public class PigStats {
return spillCount;
}
+ public long getProactiveSpillCount() {
+ long spillCount = 0;
+ for (String jid : rootJobIDs) {
+ Map<String, String> jobStats = stats.get(jid);
+ if (jobStats == null) continue;
+ if (Long.parseLong(jobStats.get("PIG_STATS_PROACTIVE_SPILL_COUNT"))==-1L)
+ {
+ spillCount = -1L;
+ break;
+ }
+ spillCount += Long.parseLong(jobStats.get("PIG_STATS_PROACTIVE_SPILL_COUNT"));
+ }
+ return spillCount;
+ }
+
private long getLocalBytesWritten() {
for(PhysicalOperator op : php.getLeaves())
return Long.parseLong(stats.get(op.toString()).get("PIG_STATS_LOCAL_BYTES_WRITTEN"));