You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ga...@apache.org on 2016/10/11 13:05:57 UTC
kylin git commit: KYLIN-2083 more RAM estimation test for
MeasureAggregator and GTAggregateScanner
Repository: kylin
Updated Branches:
refs/heads/master ff8510313 -> 460536c00
KYLIN-2083 more RAM estimation test for MeasureAggregator and GTAggregateScanner
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/460536c0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/460536c0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/460536c0
Branch: refs/heads/master
Commit: 460536c001575bd09e9f059062efe0d1049d6b31
Parents: ff85103
Author: gaodayue <ga...@meituan.com>
Authored: Wed Sep 28 17:09:06 2016 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Tue Oct 11 20:36:11 2016 +0800
----------------------------------------------------------------------
core-cube/pom.xml | 5 +
.../gridtable/AggregationCacheMemSizeTest.java | 290 +++++++++----------
core-metadata/pom.xml | 5 +
.../measure/AggregatorMemEstimateTest.java | 104 +++++++
pom.xml | 35 ++-
5 files changed, 292 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-cube/pom.xml
----------------------------------------------------------------------
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 7c9a549..39bba59 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -69,6 +69,11 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.github.jbellis</groupId>
+ <artifactId>jamm</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index fcd434e..00c0bd0 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -17,197 +17,195 @@
package org.apache.kylin.gridtable;
-import java.math.BigDecimal;
-import java.util.Comparator;
-import java.util.Random;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
+import com.google.common.base.Stopwatch;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.basic.BigDecimalSumAggregator;
import org.apache.kylin.measure.basic.DoubleSumAggregator;
import org.apache.kylin.measure.basic.LongSumAggregator;
+import org.apache.kylin.measure.bitmap.BitmapAggregator;
+import org.apache.kylin.measure.bitmap.BitmapCounter;
import org.apache.kylin.measure.hllc.HLLCAggregator;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.metadata.datatype.DoubleMutable;
import org.apache.kylin.metadata.datatype.LongMutable;
+import org.github.jamm.MemoryMeter;
import org.junit.Test;
+import java.math.BigDecimal;
+import java.util.*;
+
public class AggregationCacheMemSizeTest {
+ private static final MemoryMeter meter = new MemoryMeter();
+ private static final BitmapCounter[] bitmaps = new BitmapCounter[5];
+ private static final Random random = new Random();
+
+ // consider bitmaps with variant cardinality
+ static {
+ for (int i = 0; i < bitmaps.length; i++) {
+ bitmaps[i] = new BitmapCounter();
+ }
- public static final int NUM_OF_OBJS = 1000000 / 2;
+ final int totalBits = 1_000_000;
- interface CreateAnObject {
- Object create();
- }
+ // case 0: sparse, low-cardinality bitmap
+ for (int i = 0; i < 100; i++) {
+ bitmaps[0].add(random.nextInt(totalBits));
+ }
- @Test
- public void testHLLCAggregatorSize() throws InterruptedException {
- int est = estimateObjectSize(new CreateAnObject() {
- @Override
- public Object create() {
- HLLCAggregator aggr = new HLLCAggregator(10);
- aggr.aggregate(new HyperLogLogPlusCounter(10));
- return aggr;
+ // case 1: 20% full bitmap
+ for (int i = 0; i < totalBits; i++) {
+ if (random.nextInt(100) < 20) {
+ bitmaps[1].add(i);
}
- });
- System.out.println("HLLC: " + est);
- }
+ }
- @Test
- public void testBigDecimalAggregatorSize() throws InterruptedException {
- int est = estimateObjectSize(new CreateAnObject() {
- @Override
- public Object create() {
- return newBigDecimalAggr();
+ // case 2: half full bitmap
+ for (int i = 0; i < totalBits; i++) {
+ if (random.nextInt(100) < 50) {
+ bitmaps[2].add(i);
}
+ }
- });
- System.out.println("BigDecimal: " + est);
- }
+ // case 3: 80% full bitmap
+ for (int i = 0; i < totalBits; i++) {
+ if (random.nextInt(100) < 80) {
+ bitmaps[3].add(i);
+ }
+ }
- private BigDecimalSumAggregator newBigDecimalAggr() {
- BigDecimalSumAggregator aggr = new BigDecimalSumAggregator();
- aggr.aggregate(new BigDecimal("12345678901234567890.123456789"));
- return aggr;
+ // case 4: dense, high-cardinality bitmap
+ for (int i = 0; i < totalBits; i++) {
+ if (random.nextInt(totalBits) < 100) {
+ continue;
+ }
+ bitmaps[4].add(i);
+ }
}
- @Test
- public void testLongAggregatorSize() throws InterruptedException {
- int est = estimateObjectSize(new CreateAnObject() {
- @Override
- public Object create() {
- return newLongAggr();
- }
- });
- System.out.println("Long: " + est);
+ enum Settings {
+ WITHOUT_MEM_HUNGRY, // only test basic aggrs
+ WITH_HLLC, // basic aggrs + hllc
+ WITH_LOW_CARD_BITMAP, // basic aggrs + bitmap
+ WITH_HIGH_CARD_BITMAP // basic aggrs + bitmap
}
- private LongSumAggregator newLongAggr() {
- LongSumAggregator aggr = new LongSumAggregator();
- aggr.aggregate(new LongMutable(10));
- return aggr;
+ private MeasureAggregator<?>[] createNoMemHungryAggrs() {
+ LongSumAggregator longSum = new LongSumAggregator();
+ longSum.aggregate(new LongMutable(10));
+
+ DoubleSumAggregator doubleSum = new DoubleSumAggregator();
+ doubleSum.aggregate(new DoubleMutable(10));
+
+ BigDecimalSumAggregator decimalSum = new BigDecimalSumAggregator();
+ decimalSum.aggregate(new BigDecimal("12345678901234567890.123456789"));
+
+ return new MeasureAggregator[] { longSum, doubleSum, decimalSum };
}
- @Test
- public void testDoubleAggregatorSize() throws InterruptedException {
- int est = estimateObjectSize(new CreateAnObject() {
- @Override
- public Object create() {
- return newDoubleAggr();
- }
- });
- System.out.println("Double: " + est);
+ private HLLCAggregator createHLLCAggr() {
+ HLLCAggregator hllcAggregator = new HLLCAggregator(14);
+ hllcAggregator.aggregate(new HyperLogLogPlusCounter(14));
+ return hllcAggregator;
}
- private DoubleSumAggregator newDoubleAggr() {
- DoubleSumAggregator aggr = new DoubleSumAggregator();
- aggr.aggregate(new DoubleMutable(10));
- return aggr;
+ private BitmapAggregator createBitmapAggr(boolean lowCardinality) {
+ BitmapCounter counter = new BitmapCounter();
+ counter.merge(lowCardinality ? bitmaps[0] : bitmaps[3]);
+
+ BitmapAggregator result = new BitmapAggregator();
+ result.aggregate(counter);
+ return result;
}
- @Test
- public void testByteArraySize() throws InterruptedException {
- int est = estimateObjectSize(new CreateAnObject() {
- @Override
- public Object create() {
- return new byte[10];
- }
- });
- System.out.println("byte[10]: " + est);
+ private MeasureAggregator<?>[] createAggrs(Settings settings) {
+ List<MeasureAggregator<?>> aggregators = new ArrayList<>();
+ aggregators.addAll(Arrays.asList(createNoMemHungryAggrs()));
+
+ switch (settings) {
+ case WITHOUT_MEM_HUNGRY:
+ break;
+ case WITH_HLLC:
+ aggregators.add(createHLLCAggr());
+ break;
+ case WITH_LOW_CARD_BITMAP:
+ aggregators.add(createBitmapAggr(true));
+ break;
+ case WITH_HIGH_CARD_BITMAP:
+ aggregators.add(createBitmapAggr(false));
+ break;
+ }
+
+ return aggregators.toArray(new MeasureAggregator[aggregators.size()]);
}
@Test
- public void testAggregatorArraySize() throws InterruptedException {
- int est = estimateObjectSize(new CreateAnObject() {
- @Override
- public Object create() {
- return new MeasureAggregator[7];
- }
- });
- System.out.println("MeasureAggregator[7]: " + est);
+ public void testEstimateBitmapMemSize() {
+ BitmapAggregator[] bitmapAggrs = new BitmapAggregator[bitmaps.length];
+ for (int i = 0; i < bitmapAggrs.length; i++) {
+ bitmapAggrs[i] = new BitmapAggregator();
+ bitmapAggrs[i].aggregate(bitmaps[i]);
+ }
+
+ System.out.printf("%-15s %-10s %-10s\n", "cardinality", "estimate", "actual");
+ for (BitmapAggregator aggr : bitmapAggrs) {
+ System.out.printf("%-15d %-10d %-10d\n",
+ aggr.getState().getCount(), aggr.getMemBytesEstimate(), meter.measureDeep(aggr));
+ }
}
@Test
- public void testTreeMapSize() throws InterruptedException {
- final SortedMap<byte[], Object> map = new TreeMap<byte[], Object>(new Comparator<byte[]>() {
- @Override
- public int compare(byte[] o1, byte[] o2) {
- return Bytes.compareTo(o1, o2);
- }
- });
- final Random rand = new Random();
- int est = estimateObjectSize(new CreateAnObject() {
- @Override
- public Object create() {
- byte[] key = new byte[10];
- rand.nextBytes(key);
- map.put(key, null);
- return null;
- }
- });
- System.out.println("TreeMap entry: " + (est - 20)); // -20 is to exclude byte[10]
+ public void testEstimateMemSize() throws InterruptedException {
+ int scale = Integer.parseInt(System.getProperty("scale", "1"));
+ scale = Math.max(1, Math.min(10, scale));
+
+ testSetting(Settings.WITHOUT_MEM_HUNGRY, scale * 100000);
+ testSetting(Settings.WITH_HLLC, scale * 5000);
+ testSetting(Settings.WITH_LOW_CARD_BITMAP, scale * 10000);
+ testSetting(Settings.WITH_HIGH_CARD_BITMAP, scale * 1000);
}
- @Test
- public void testAggregationCacheSize() throws InterruptedException {
- final SortedMap<byte[], Object> map = new TreeMap<byte[], Object>(new Comparator<byte[]>() {
+ private void testSetting(Settings settings, int inputCount) {
+ SortedMap<byte[], Object> map = new TreeMap<>(new Comparator<byte[]>() {
@Override
public int compare(byte[] o1, byte[] o2) {
return Bytes.compareTo(o1, o2);
}
});
- final Random rand = new Random();
-
- long bytesBefore = memLeft();
- byte[] key = null;
- MeasureAggregator<?>[] aggrs = null;
- for (int i = 0; i < NUM_OF_OBJS; i++) {
- key = new byte[10];
- rand.nextBytes(key);
- aggrs = new MeasureAggregator[4];
- aggrs[0] = newBigDecimalAggr();
- aggrs[1] = newLongAggr();
- aggrs[2] = newDoubleAggr();
- aggrs[3] = newDoubleAggr();
- map.put(key, aggrs);
- }
-
- long bytesAfter = memLeft();
-
- long mapActualSize = bytesBefore - bytesAfter;
- long mapExpectSize = GTAggregateScanner.estimateSizeOfAggrCache(key, aggrs, map.size());
- System.out.println("Actual cache size: " + mapActualSize);
- System.out.println("Expect cache size: " + mapExpectSize);
- }
-
- private int estimateObjectSize(CreateAnObject factory) throws InterruptedException {
- Object[] hold = new Object[NUM_OF_OBJS];
- long bytesBefore = memLeft();
- for (int i = 0; i < hold.length; i++) {
- hold[i] = factory.create();
+ final int reportInterval = inputCount / 10;
+ final Stopwatch stopwatch = new Stopwatch();
+ long estimateMillis = 0;
+ long actualMillis = 0;
+
+ System.out.println("Settings: " + settings);
+ System.out.printf("%15s %15s %15s %15s %15s\n",
+ "Size", "Estimate(bytes)", "Actual(bytes)", "Estimate(ms)", "Actual(ms)");
+
+ for (int i = 0; i < inputCount; i++) {
+ byte[] key = new byte[10];
+ random.nextBytes(key);
+ MeasureAggregator[] values = createAggrs(settings);
+ map.put(key, values);
+
+ if ((i+1) % reportInterval == 0) {
+ stopwatch.start();
+ long estimateBytes = GTAggregateScanner.estimateSizeOfAggrCache(key, values, map.size());
+ estimateMillis += stopwatch.elapsedMillis();
+ stopwatch.reset();
+
+ stopwatch.start();
+ long actualBytes = meter.measureDeep(map);
+ actualMillis += stopwatch.elapsedMillis();
+ stopwatch.reset();
+
+ System.out.printf("%,15d %,15d %,15d %,15d %,15d\n",
+ map.size(), estimateBytes, actualBytes, estimateMillis, actualMillis);
+ }
}
+ System.out.println("---------------------------------------\n");
- long bytesAfter = memLeft();
- return (int) ((bytesBefore - bytesAfter) / hold.length);
- }
-
- private long memLeft() throws InterruptedException {
- Runtime.getRuntime().gc();
- Thread.sleep(500);
- return getSystemAvailBytes();
- }
-
- private long getSystemAvailBytes() {
- Runtime runtime = Runtime.getRuntime();
- long totalMemory = runtime.totalMemory(); // current heap allocated to the VM process
- long freeMemory = runtime.freeMemory(); // out of the current heap, how much is free
- long maxMemory = runtime.maxMemory(); // Max heap VM can use e.g. Xmx setting
- long usedMemory = totalMemory - freeMemory; // how much of the current heap the VM is using
- long availableMemory = maxMemory - usedMemory; // available memory i.e. Maximum heap size minus the current amount used
- return availableMemory;
+ map = null;
+ System.gc();
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 142dd33..c95f7f0 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -65,6 +65,11 @@
</dependency>
<dependency>
+ <groupId>com.github.jbellis</groupId>
+ <artifactId>jamm</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
new file mode 100644
index 0000000..2883923
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
@@ -0,0 +1,104 @@
+package org.apache.kylin.measure;
+
+import com.google.common.collect.Lists;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.measure.basic.*;
+import org.apache.kylin.measure.bitmap.BitmapAggregator;
+import org.apache.kylin.measure.bitmap.BitmapCounter;
+import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
+import org.apache.kylin.measure.hllc.HLLCAggregator;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DoubleMutable;
+import org.apache.kylin.metadata.datatype.LongMutable;
+import org.github.jamm.MemoryMeter;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase {
+ private static final MemoryMeter meter = new MemoryMeter();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ staticCreateTestMetadata();
+ }
+
+ @AfterClass
+ public static void after() throws Exception {
+ cleanAfterClass();
+ }
+
+ private List<? extends MeasureAggregator> basicAggregators() {
+ LongMutable longVal = new LongMutable(1000);
+ LongMinAggregator longMin = new LongMinAggregator();
+ LongMaxAggregator longMax = new LongMaxAggregator();
+ LongSumAggregator longSum = new LongSumAggregator();
+ longMin.aggregate(longVal);
+ longMax.aggregate(longVal);
+ longSum.aggregate(longVal);
+
+ DoubleMutable doubleVal = new DoubleMutable(1.0);
+ DoubleMinAggregator doubleMin = new DoubleMinAggregator();
+ DoubleMaxAggregator doubleMax = new DoubleMaxAggregator();
+ DoubleSumAggregator doubleSum = new DoubleSumAggregator();
+ doubleMin.aggregate(doubleVal);
+ doubleMax.aggregate(doubleVal);
+ doubleSum.aggregate(doubleVal);
+
+ BigDecimalMinAggregator decimalMin = new BigDecimalMinAggregator();
+ BigDecimalMaxAggregator decimalMax = new BigDecimalMaxAggregator();
+ BigDecimalSumAggregator decimalSum = new BigDecimalSumAggregator();
+ BigDecimal decimal = new BigDecimal("12345678901234567890.123456789");
+ decimalMin.aggregate(decimal);
+ decimalMax.aggregate(decimal);
+ decimalSum.aggregate(decimal);
+
+ return Lists.newArrayList(
+ longMin, longMax, longSum,
+ doubleMin, doubleMax, doubleSum,
+ decimalMin, decimalMax, decimalSum
+ );
+ }
+
+ private String getAggregatorName(Class<? extends MeasureAggregator> clazz) {
+ if (!clazz.isAnonymousClass()) {
+ return clazz.getSimpleName();
+ }
+ String[] parts = clazz.getName().split("\\.");
+ return parts[parts.length - 1];
+ }
+
+ @Test
+ public void testAggregatorEstimate() {
+ HLLCAggregator hllcAggregator = new HLLCAggregator(14);
+ hllcAggregator.aggregate(new HyperLogLogPlusCounter(14));
+
+ BitmapAggregator bitmapAggregator = new BitmapAggregator();
+ BitmapCounter bitmapCounter = new BitmapCounter();
+ for (int i = 4000; i <= 100000; i += 2) {
+ bitmapCounter.add(i);
+ }
+ bitmapAggregator.aggregate(bitmapCounter);
+
+ ExtendedColumnMeasureType extendedColumnType = new ExtendedColumnMeasureType("EXTENDED_COLUMN", DataType.getType("extendedcolumn(100)"));
+ MeasureAggregator<ByteArray> extendedColumnAggregator = extendedColumnType.newAggregator();
+ extendedColumnAggregator.aggregate(new ByteArray(100));
+
+ List<MeasureAggregator> aggregators = Lists.newArrayList(basicAggregators());
+ aggregators.add(hllcAggregator);
+ aggregators.add(bitmapAggregator);
+ aggregators.add(extendedColumnAggregator);
+
+ System.out.printf("%40s %10s %10s\n", "Class", "Estimate", "Actual");
+ for (MeasureAggregator aggregator : aggregators) {
+ String clzName = getAggregatorName(aggregator.getClass());
+ System.out.printf("%40s %10d %10d\n", clzName, aggregator.getMemBytesEstimate(), meter.measureDeep(aggregator));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/460536c0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 72e4069..caa09ec 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,6 +72,7 @@
<dbunit.version>2.5.2</dbunit.version>
<h2.version>1.4.192</h2.version>
<jetty.version>9.3.10.v20160621</jetty.version>
+ <jamm.version>0.3.1</jamm.version>
<!-- Commons -->
<commons-lang.version>2.6</commons-lang.version>
@@ -122,6 +123,8 @@
org/apache/kylin/**/tools/**:**/*CLI.java
</sonar.jacoco.excludes>
+ <!-- JVM Args for Testing -->
+ <argLine>-Xms1G -Xmx2G -XX:MaxPermSize=512M</argLine>
</properties>
<licenses>
@@ -635,6 +638,11 @@
<artifactId>kryo-shaded</artifactId>
<version>${kryo.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.github.jbellis</groupId>
+ <artifactId>jamm</artifactId>
+ <version>${jamm.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.curator</groupId>
@@ -869,6 +877,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
+ <version>2.10</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -1046,6 +1055,30 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-jamm</id>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <phase>generate-test-resources</phase>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>com.github.jbellis</groupId>
+ <artifactId>jamm</artifactId>
+ <outputDirectory>${project.build.testOutputDirectory}</outputDirectory>
+ <destFileName>jamm.jar</destFileName>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.19.1</version>
<configuration>
@@ -1067,7 +1100,7 @@
<value>kylin-log4j.properties</value>
</property>
</systemProperties>
- <argLine>-Xms1G -Xmx2G -XX:MaxPermSize=512M</argLine>
+ <argLine>-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine}</argLine>
</configuration>
</plugin>