You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2016/02/10 09:10:14 UTC
[1/2] hbase git commit: HBASE-15171 Avoid counting duplicate kv and
generating lots of small hfiles in PutSortReducer (Yu Li)
Repository: hbase
Updated Branches:
refs/heads/0.98 6bf47a030 -> de149d0bc
HBASE-15171 Avoid counting duplicate kv and generating lots of small hfiles in PutSortReducer (Yu Li)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/38cd179b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/38cd179b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/38cd179b
Branch: refs/heads/0.98
Commit: 38cd179bb540f0d38c5810a17097c5727947ca73
Parents: 6bf47a0
Author: tedyu <yu...@gmail.com>
Authored: Wed Jan 27 09:49:45 2016 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Feb 10 00:07:34 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/38cd179b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
index db9e585..dbf4e30 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
@@ -67,9 +67,11 @@ public class PutSortReducer extends
for (Cell cell: cells) {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
map.add(kv);
- curSize += kv.heapSize();
}
}
+ for(KeyValue kv: map){
+ curSize +=kv.heapSize();
+ }
}
context.setStatus("Read " + map.size() + " entries of " + map.getClass()
+ "(" + StringUtils.humanReadableInt(curSize) + ")");
[2/2] hbase git commit: HBASE-15171 Addendum removes extra loop (Yu
Li)
Posted by ap...@apache.org.
HBASE-15171 Addendum removes extra loop (Yu Li)
Amending-Author: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/de149d0b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/de149d0b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/de149d0b
Branch: refs/heads/0.98
Commit: de149d0bc4eda960e7246c79a1ad85c9cbe50de0
Parents: 38cd179
Author: tedyu <yu...@gmail.com>
Authored: Thu Jan 28 07:08:03 2016 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Feb 10 00:08:55 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/mapreduce/PutSortReducer.java | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/de149d0b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
index dbf4e30..792686a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
@@ -66,12 +66,11 @@ public class PutSortReducer extends
for (List<Cell> cells: p.getFamilyCellMap().values()) {
for (Cell cell: cells) {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
- map.add(kv);
+ if (map.add(kv)) {// don't count duplicated kv into size
+ curSize += kv.heapSize();
+ }
}
}
- for(KeyValue kv: map){
- curSize +=kv.heapSize();
- }
}
context.setStatus("Read " + map.size() + " entries of " + map.getClass()
+ "(" + StringUtils.humanReadableInt(curSize) + ")");