You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/05/29 11:58:39 UTC
incubator-kylin git commit: KYLIN-798,
estimate AggregationCache size correctly
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8.0 49362dd14 -> a768f1900
KYLIN-798, estimate AggregationCache size correctly
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a768f190
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a768f190
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a768f190
Branch: refs/heads/0.8.0
Commit: a768f1900b7c199577807759732a2c367ceb32a3
Parents: 49362dd
Author: Li, Yang <ya...@ebay.com>
Authored: Fri May 29 17:56:35 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri May 29 17:57:26 2015 +0800
----------------------------------------------------------------------
.../common/hll/HyperLogLogPlusCounter.java | 4 -
.../common/util/MemoryBudgetController.java | 9 +-
.../measure/BigDecimalMaxAggregator.java | 2 +-
.../measure/BigDecimalMinAggregator.java | 2 +-
.../measure/BigDecimalSumAggregator.java | 2 +-
.../metadata/measure/DoubleMaxAggregator.java | 2 +-
.../metadata/measure/DoubleMinAggregator.java | 2 +-
.../metadata/measure/DoubleSumAggregator.java | 2 +-
.../kylin/metadata/measure/HLLCAggregator.java | 17 +-
.../kylin/metadata/measure/LDCAggregator.java | 2 +-
.../metadata/measure/LongMaxAggregator.java | 2 +-
.../metadata/measure/LongMinAggregator.java | 2 +-
.../metadata/measure/LongSumAggregator.java | 2 +-
.../metadata/measure/MeasureAggregator.java | 36 +++-
.../storage/gridtable/GTAggregateScanner.java | 29 ++-
.../hbase/coprocessor/AggregationCache.java | 2 +-
.../gridtable/AggregationCacheMemSizeTest.java | 188 +++++++++++++++++++
17 files changed, 269 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
index f4678c9..251aed1 100644
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@ -108,10 +108,6 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
return new HLLCSnapshot(this).getCountEstimate();
}
- public int getMemBytes() {
- return 12 + m;
- }
-
public double getErrorRate() {
return 1.04 / Math.sqrt(m);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java b/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index 06d3cd2..26286cb 100644
--- a/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ b/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -124,15 +124,18 @@ public class MemoryBudgetController {
return getSystemAvailMB() - SYSTEM_RESERVED >= mb;
}
- public static int getSystemAvailMB() {
+ public static long getSystemAvailBytes() {
Runtime runtime = Runtime.getRuntime();
long totalMemory = runtime.totalMemory(); // current heap allocated to the VM process
long freeMemory = runtime.freeMemory(); // out of the current heap, how much is free
long maxMemory = runtime.maxMemory(); // Max heap VM can use e.g. Xmx setting
long usedMemory = totalMemory - freeMemory; // how much of the current heap the VM is using
long availableMemory = maxMemory - usedMemory; // available memory i.e. Maximum heap size minus the current amount used
- int availMB = (int) (availableMemory / ONE_MB);
- return availMB;
+ return availableMemory;
+ }
+
+ public static int getSystemAvailMB() {
+ return (int) (getSystemAvailBytes() / ONE_MB);
}
public static int getMaxPossibleBudget() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMaxAggregator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMaxAggregator.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMaxAggregator.java
index cdc3eac..562fade 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMaxAggregator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMaxAggregator.java
@@ -47,7 +47,7 @@ public class BigDecimalMaxAggregator extends MeasureAggregator<BigDecimal> {
}
@Override
- public int getMemBytes() {
+ public int getMemBytesEstimate() {
return guessBigDecimalMemBytes();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java
index 8ad3b58..8e11fd9 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java
@@ -47,7 +47,7 @@ public class BigDecimalMinAggregator extends MeasureAggregator<BigDecimal> {
}
@Override
- public int getMemBytes() {
+ public int getMemBytesEstimate() {
return guessBigDecimalMemBytes();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java
index 2a8da82..1b2aae1 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java
@@ -44,7 +44,7 @@ public class BigDecimalSumAggregator extends MeasureAggregator<BigDecimal> {
}
@Override
- public int getMemBytes() {
+ public int getMemBytesEstimate() {
return guessBigDecimalMemBytes();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMaxAggregator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMaxAggregator.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMaxAggregator.java
index 718e8b0..1e5bf82 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMaxAggregator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMaxAggregator.java
@@ -47,7 +47,7 @@ public class DoubleMaxAggregator extends MeasureAggregator<DoubleWritable> {
}
@Override
- public int getMemBytes() {
+ public int getMemBytesEstimate() {
return guessDoubleMemBytes();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMinAggregator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMinAggregator.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMinAggregator.java
index dd37f28..d9112f7 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMinAggregator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMinAggregator.java
@@ -47,7 +47,7 @@ public class DoubleMinAggregator extends MeasureAggregator<DoubleWritable> {
}
@Override
- public int getMemBytes() {
+ public int getMemBytesEstimate() {
return guessDoubleMemBytes();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSumAggregator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSumAggregator.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSumAggregator.java
index df9722f..923bd0b 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSumAggregator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSumAggregator.java
@@ -44,7 +44,7 @@ public class DoubleSumAggregator extends MeasureAggregator<DoubleWritable> {
}
@Override
- public int getMemBytes() {
+ public int getMemBytesEstimate() {
return guessDoubleMemBytes();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCAggregator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCAggregator.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCAggregator.java
index df6938e..b07fd80 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCAggregator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCAggregator.java
@@ -26,8 +26,13 @@ import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
*/
public class HLLCAggregator extends MeasureAggregator<HyperLogLogPlusCounter> {
+ final int precision;
HyperLogLogPlusCounter sum = null;
+ public HLLCAggregator(int precision) {
+ this.precision = precision;
+ }
+
@Override
public void reset() {
sum = null;
@@ -47,11 +52,13 @@ public class HLLCAggregator extends MeasureAggregator<HyperLogLogPlusCounter> {
}
@Override
- public int getMemBytes() {
- if (sum == null)
- return Integer.MIN_VALUE;
- else
- return 4 + sum.getMemBytes();
+ public int getMemBytesEstimate() {
+ // 1024 + 60 returned by AggregationCacheMemSizeTest
+ return 8 // aggregator obj shell
+ + 4 // precision
+ + 8 // ref to HLLC
+ + 8 // HLLC obj shell
+ + 32 + (1 >> precision); // HLLC internal
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/metadata/src/main/java/org/apache/kylin/metadata/measure/LDCAggregator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/LDCAggregator.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/LDCAggregator.java
index c27a150..767a6d8 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/LDCAggregator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/LDCAggregator.java
@@ -57,7 +57,7 @@ public class LDCAggregator extends MeasureAggregator<LongWritable> {
}
@Override
- public int getMemBytes() {
+ public int getMemBytesEstimate() {
return guessLongMemBytes();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/metadata/src/main/java/org/apache/kylin/metadata/measure/LongMaxAggregator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/LongMaxAggregator.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/LongMaxAggregator.java
index f6e1aea..0fac3c7 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/LongMaxAggregator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/LongMaxAggregator.java
@@ -47,7 +47,7 @@ public class LongMaxAggregator extends MeasureAggregator<LongWritable> {
}
@Override
- public int getMemBytes() {
+ public int getMemBytesEstimate() {
return guessLongMemBytes();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/metadata/src/main/java/org/apache/kylin/metadata/measure/LongMinAggregator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/LongMinAggregator.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/LongMinAggregator.java
index 6606b63..4c058d8 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/LongMinAggregator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/LongMinAggregator.java
@@ -47,7 +47,7 @@ public class LongMinAggregator extends MeasureAggregator<LongWritable> {
}
@Override
- public int getMemBytes() {
+ public int getMemBytesEstimate() {
return guessLongMemBytes();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSumAggregator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSumAggregator.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSumAggregator.java
index 535c7dc..e40870d 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSumAggregator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSumAggregator.java
@@ -44,7 +44,7 @@ public class LongSumAggregator extends MeasureAggregator<LongWritable> {
}
@Override
- public int getMemBytes() {
+ public int getMemBytesEstimate() {
return guessLongMemBytes();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java
index 0e2fc55..4153cbd 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java
@@ -36,8 +36,9 @@ abstract public class MeasureAggregator<V> {
else if (isDouble(returnType))
return new DoubleSumAggregator();
} else if (FunctionDesc.FUNC_COUNT_DISTINCT.equalsIgnoreCase(funcName)) {
- if (DataType.getInstance(returnType).isHLLC())
- return new HLLCAggregator();
+ DataType hllcType = DataType.getInstance(returnType);
+ if (hllcType.isHLLC())
+ return new HLLCAggregator(hllcType.getPrecision());
else
return new LDCAggregator();
} else if (FunctionDesc.FUNC_MAX.equalsIgnoreCase(funcName)) {
@@ -71,18 +72,33 @@ abstract public class MeasureAggregator<V> {
}
public static int guessBigDecimalMemBytes() {
- return 4 // ref
- + 20; // guess of BigDecimal
+ // 116 returned by AggregationCacheMemSizeTest
+ return 8 // aggregator obj shell
+ + 8 // ref to BigDecimal
+ + 8 // BigDecimal obj shell
+ + 100; // guess of BigDecimal internal
}
public static int guessDoubleMemBytes() {
- return 4 // ref
- + 8;
+ // 29 to 44 returned by AggregationCacheMemSizeTest
+ return 44;
+ /*
+ return 8 // aggregator obj shell
+ + 8 // ref to DoubleWritable
+ + 8 // DoubleWritable obj shell
+ + 8; // size of double
+ */
}
public static int guessLongMemBytes() {
- return 4 // ref
- + 8;
+ // 29 to 44 returned by AggregationCacheMemSizeTest
+ return 44;
+ /*
+ return 8 // aggregator obj shell
+ + 8 // ref to LongWritable
+ + 8 // LongWritable obj shell
+ + 8; // size of long
+ */
}
// ============================================================================
@@ -97,6 +113,6 @@ abstract public class MeasureAggregator<V> {
abstract public V getState();
- // get an estimate of memory consumption
- abstract public int getMemBytes();
+ // get an estimate of memory consumption UPPER BOUND
+ abstract public int getMemBytesEstimate();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
index ef41e05..a7110ef 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
@@ -64,7 +64,7 @@ public class GTAggregateScanner implements IGTScanner {
@Override
public Iterator<GTRecord> iterator() {
- AggregationCacheWithBytesKey aggregationCacheWithBytesKey = new AggregationCacheWithBytesKey();
+ AggregationCache aggregationCacheWithBytesKey = new AggregationCache();
for (GTRecord r : inputScanner) {
aggregationCacheWithBytesKey.aggregate(r);
MemoryChecker.checkMemory();
@@ -72,12 +72,12 @@ public class GTAggregateScanner implements IGTScanner {
return aggregationCacheWithBytesKey.iterator();
}
- class AggregationCacheWithBytesKey {
+ class AggregationCache {
final SortedMap<byte[], MeasureAggregator[]> aggBufMap;
final int keyLength;
final boolean[] compareMask;
- public AggregationCacheWithBytesKey() {
+ public AggregationCache() {
compareMask = createCompareMask();
keyLength = compareMask.length;
aggBufMap = Maps.newTreeMap(new Comparator<byte[]>() {
@@ -194,4 +194,27 @@ public class GTAggregateScanner implements IGTScanner {
}
}
+ public static long estimateSizeOfAggrCache(byte[] keySample, MeasureAggregator<?>[] aggrSample, int size) {
+ // Aggregation cache is basically a tree map. The tree map entry overhead is
+ // - 40 according to http://java-performance.info/memory-consumption-of-java-data-types-2/
+ // - 41~52 according to AggregationCacheMemSizeTest
+ return (estimateSizeOf(keySample) + estimateSizeOf(aggrSample) + 64) * size;
+ }
+
+ public static long estimateSizeOf(MeasureAggregator[] aggrs) {
+ // size of array, AggregationCacheMemSizeTest reports 4 for [0], 12 for [1], 12 for [2], 20 for [3] etc..
+ // Memory alignment to 8 bytes
+ long est = (aggrs.length + 1) / 2 * 8 + 4 + (4 /* extra */);
+ for (MeasureAggregator aggr : aggrs) {
+ if (aggr != null)
+ est += aggr.getMemBytesEstimate();
+ }
+ return est;
+ }
+
+ public static long estimateSizeOf(byte[] bytes) {
+ // AggregationCacheMemSizeTest reports 20 for byte[10] and 20 again for byte[16]
+ // Memory alignment to 8 bytes
+ return (bytes.length + 7) / 8 * 8 + 4 + (4 /* extra */);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
index 4274b9d..2553dfd 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
@@ -68,7 +68,7 @@ public abstract class AggregationCache {
rowMemBytes = 0;
MeasureAggregator[] measureAggregators = aggBufMap.get(firstKey);
for (MeasureAggregator agg : measureAggregators) {
- rowMemBytes += agg.getMemBytes();
+ rowMemBytes += agg.getMemBytesEstimate();
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a768f190/storage/src/test/java/org/apache/kylin/storage/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/AggregationCacheMemSizeTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/AggregationCacheMemSizeTest.java
new file mode 100644
index 0000000..52c092e
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/gridtable/AggregationCacheMemSizeTest.java
@@ -0,0 +1,188 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.math.BigDecimal;
+import java.util.Comparator;
+import java.util.Random;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.MemoryBudgetController;
+import org.apache.kylin.metadata.measure.BigDecimalSumAggregator;
+import org.apache.kylin.metadata.measure.DoubleSumAggregator;
+import org.apache.kylin.metadata.measure.HLLCAggregator;
+import org.apache.kylin.metadata.measure.LongSumAggregator;
+import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.junit.Test;
+
+/** Note: Execute each test alone to get accurate size estimate. */
+public class AggregationCacheMemSizeTest {
+
+ public static final int NUM_OF_OBJS = 1000000 / 2;
+
+ interface CreateAnObject {
+ Object create();
+ }
+
+ @Test
+ public void testHLLCAggregatorSize() throws InterruptedException {
+ int est = estimateObjectSize(new CreateAnObject() {
+ @Override
+ public Object create() {
+ HLLCAggregator aggr = new HLLCAggregator(10);
+ aggr.aggregate(new HyperLogLogPlusCounter(10));
+ return aggr;
+ }
+ });
+ System.out.println("HLLC: " + est);
+ }
+
+ @Test
+ public void testBigDecimalAggregatorSize() throws InterruptedException {
+ int est = estimateObjectSize(new CreateAnObject() {
+ @Override
+ public Object create() {
+ return newBigDecimalAggr();
+ }
+
+ });
+ System.out.println("BigDecimal: " + est);
+ }
+
+ private BigDecimalSumAggregator newBigDecimalAggr() {
+ BigDecimalSumAggregator aggr = new BigDecimalSumAggregator();
+ aggr.aggregate(new BigDecimal("12345678901234567890.123456789"));
+ return aggr;
+ }
+
+ @Test
+ public void testLongAggregatorSize() throws InterruptedException {
+ int est = estimateObjectSize(new CreateAnObject() {
+ @Override
+ public Object create() {
+ return newLongAggr();
+ }
+ });
+ System.out.println("Long: " + est);
+ }
+
+ private LongSumAggregator newLongAggr() {
+ LongSumAggregator aggr = new LongSumAggregator();
+ aggr.aggregate(new LongWritable(10));
+ return aggr;
+ }
+
+ @Test
+ public void testDoubleAggregatorSize() throws InterruptedException {
+ int est = estimateObjectSize(new CreateAnObject() {
+ @Override
+ public Object create() {
+ return newDoubleAggr();
+ }
+ });
+ System.out.println("Double: " + est);
+ }
+
+ private DoubleSumAggregator newDoubleAggr() {
+ DoubleSumAggregator aggr = new DoubleSumAggregator();
+ aggr.aggregate(new DoubleWritable(10));
+ return aggr;
+ }
+
+ @Test
+ public void testByteArraySize() throws InterruptedException {
+ int est = estimateObjectSize(new CreateAnObject() {
+ @Override
+ public Object create() {
+ return new byte[10];
+ }
+ });
+ System.out.println("byte[10]: " + est);
+ }
+
+ @Test
+ public void testAggregatorArraySize() throws InterruptedException {
+ int est = estimateObjectSize(new CreateAnObject() {
+ @Override
+ public Object create() {
+ return new MeasureAggregator[7];
+ }
+ });
+ System.out.println("MeasureAggregator[7]: " + est);
+ }
+
+ @Test
+ public void testTreeMapSize() throws InterruptedException {
+ final SortedMap<byte[], Object> map = new TreeMap<byte[], Object>(new Comparator<byte[]>() {
+ @Override
+ public int compare(byte[] o1, byte[] o2) {
+ return Bytes.compareTo(o1, o2);
+ }
+ });
+ final Random rand = new Random();
+ int est = estimateObjectSize(new CreateAnObject() {
+ @Override
+ public Object create() {
+ byte[] key = new byte[10];
+ rand.nextBytes(key);
+ map.put(key, null);
+ return null;
+ }
+ });
+ System.out.println("TreeMap entry: " + (est - 20)); // -20 is to exclude byte[10]
+ }
+
+ @Test
+ public void testAggregationCacheSize() throws InterruptedException {
+ final SortedMap<byte[], Object> map = new TreeMap<byte[], Object>(new Comparator<byte[]>() {
+ @Override
+ public int compare(byte[] o1, byte[] o2) {
+ return Bytes.compareTo(o1, o2);
+ }
+ });
+ final Random rand = new Random();
+
+ long bytesBefore = memLeft();
+ byte[] key = null;
+ MeasureAggregator<?>[] aggrs = null;
+ for (int i = 0; i < NUM_OF_OBJS; i++) {
+ key = new byte[10];
+ rand.nextBytes(key);
+ aggrs = new MeasureAggregator[4];
+ aggrs[0] = newBigDecimalAggr();
+ aggrs[1] = newLongAggr();
+ aggrs[2] = newDoubleAggr();
+ aggrs[3] = newDoubleAggr();
+ map.put(key, aggrs);
+ }
+
+ long bytesAfter = memLeft();
+
+ long mapActualSize = bytesBefore - bytesAfter;
+ long mapExpectSize = GTAggregateScanner.estimateSizeOfAggrCache(key, aggrs, map.size());
+ System.out.println("Actual cache size: " + mapActualSize);
+ System.out.println("Expect cache size: " + mapExpectSize);
+ }
+
+ private int estimateObjectSize(CreateAnObject factory) throws InterruptedException {
+ Object[] hold = new Object[NUM_OF_OBJS];
+ long bytesBefore = memLeft();
+
+ for (int i = 0; i < hold.length; i++) {
+ hold[i] = factory.create();
+ }
+
+ long bytesAfter = memLeft();
+ return (int) ((bytesBefore - bytesAfter) / hold.length);
+ }
+
+ private long memLeft() throws InterruptedException {
+ Runtime.getRuntime().gc();
+ Thread.sleep(500);
+ return MemoryBudgetController.getSystemAvailBytes();
+ }
+
+}