You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by sy...@apache.org on 2016/01/29 18:13:55 UTC
[02/13] hbase git commit: HBASE-15171 Avoid counting duplicate kv and
generating lots of small hfiles in PutSortReducer (Yu Li)
HBASE-15171 Avoid counting duplicate kv and generating lots of small hfiles in PutSortReducer (Yu Li)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/47c41479
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/47c41479
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/47c41479
Branch: refs/heads/hbase-12439
Commit: 47c41479401ea0aadfa3c3776fe2930bb8e9710d
Parents: 845d00a
Author: tedyu <yu...@gmail.com>
Authored: Wed Jan 27 09:42:56 2016 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Wed Jan 27 09:42:56 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/mapreduce/PutSortReducer.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/47c41479/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
index 0a27056..d18ea34 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
@@ -22,12 +22,12 @@ import java.util.Iterator;
import java.util.List;
import java.util.TreeSet;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.mapreduce.Reducer;
@@ -68,9 +68,11 @@ public class PutSortReducer extends
for (Cell cell: cells) {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
map.add(kv);
- curSize += kv.heapSize();
}
}
+ for(KeyValue kv: map){
+ curSize +=kv.heapSize();
+ }
}
context.setStatus("Read " + map.size() + " entries of " + map.getClass()
+ "(" + StringUtils.humanReadableInt(curSize) + ")");