You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by la...@apache.org on 2016/06/17 19:37:51 UTC
phoenix git commit: PHOENIX-3000 Reduce memory consumption during
DISTINCT aggregation.
Repository: phoenix
Updated Branches:
refs/heads/4.x-HBase-0.98 ac63502e0 -> 2d874c732
PHOENIX-3000 Reduce memory consumption during DISTINCT aggregation.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/2d874c73
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/2d874c73
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/2d874c73
Branch: refs/heads/4.x-HBase-0.98
Commit: 2d874c7322f54095c030501a0ac78ee27af5610e
Parents: ac63502
Author: Lars Hofhansl <la...@apache.org>
Authored: Fri Jun 17 12:37:23 2016 -0700
Committer: Lars Hofhansl <la...@apache.org>
Committed: Fri Jun 17 12:37:40 2016 -0700
----------------------------------------------------------------------
.../DistinctValueWithCountServerAggregator.java | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/2d874c73/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java
index ee9f7f2..4801a9d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java
@@ -24,6 +24,7 @@ import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.cache.GlobalCache;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
@@ -44,6 +45,11 @@ import org.iq80.snappy.Snappy;
public class DistinctValueWithCountServerAggregator extends BaseAggregator {
public static final int DEFAULT_ESTIMATED_DISTINCT_VALUES = 10000;
public static final byte[] COMPRESS_MARKER = new byte[] { (byte)1 };
+ // copy a key unless it uses at least 10% of the backing array
+ private static final int COPY_THRESHOLD = 100/10;
+ // copy key only (make a new array) if the backing array is at least this size
+ // (to avoid ending up using _more_ memory)
+ private static final int FIXED_COPY_THRESHOLD = SizedUtil.ARRAY_SIZE * 2;
private int compressThreshold;
private byte[] buffer = null;
@@ -62,7 +68,10 @@ public class DistinctValueWithCountServerAggregator extends BaseAggregator {
@Override
public void aggregate(Tuple tuple, ImmutableBytesWritable ptr) {
- ImmutableBytesPtr key = new ImmutableBytesPtr(ptr.get(), ptr.getOffset(), ptr.getLength());
+ ImmutableBytesPtr key = ptr.get().length > FIXED_COPY_THRESHOLD &&
+ ptr.get().length > ptr.getLength() * COPY_THRESHOLD ?
+ new ImmutableBytesPtr(ptr.copyBytes()) :
+ new ImmutableBytesPtr(ptr);
Integer count = this.valueVsCount.get(key);
if (count == null) {
this.valueVsCount.put(key, 1);