You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/03/24 09:51:49 UTC
kylin git commit: Update test VM settings
Repository: kylin
Updated Branches:
refs/heads/master 4adea1677 -> d2476f301
Update test VM settings
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d2476f30
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d2476f30
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d2476f30
Branch: refs/heads/master
Commit: d2476f3010ea9ff6190c64c10f16c209b694c498
Parents: 4adea16
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Mar 24 16:51:31 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Mar 24 16:51:31 2016 +0800
----------------------------------------------------------------------
.../DoggedCubeBuilderStressTest.java | 105 ------------------
kylin-it/pom.xml | 9 +-
.../ITDoggedCubeBuilderStressTest.java | 107 +++++++++++++++++++
pom.xml | 2 +-
4 files changed, 116 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d2476f30/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java
deleted file mode 100644
index 8d8366e..0000000
--- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.inmemcubing;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.dimension.Dictionary;
-import org.apache.kylin.gridtable.GTRecord;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class DoggedCubeBuilderStressTest extends LocalFileMetadataTestCase {
-
- @SuppressWarnings("unused")
- private static final Logger logger = LoggerFactory.getLogger(DoggedCubeBuilderStressTest.class);
-
- // CI sandbox memory is no more than 512MB, this many input should hit memory threshold
- private static final int INPUT_ROWS = 200000;
- private static final int THREADS = 4;
-
- private static CubeInstance cube;
- private static String flatTable;
- private static Map<TblColRef, Dictionary<String>> dictionaryMap;
-
- @BeforeClass
- public static void before() throws IOException {
- staticCreateTestMetadata();
-
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
-
- cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
- flatTable = "../examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv";
- dictionaryMap = InMemCubeBuilderTest.getDictionaryMap(cube, flatTable);
- }
-
- @AfterClass
- public static void after() throws Exception {
- staticCleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
-
- ArrayBlockingQueue<List<String>> queue = new ArrayBlockingQueue<List<String>>(1000);
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- long randSeed = System.currentTimeMillis();
-
- DoggedCubeBuilder doggedBuilder = new DoggedCubeBuilder(cube.getDescriptor(), dictionaryMap);
- doggedBuilder.setConcurrentThreads(THREADS);
-
- {
- Future<?> future = executorService.submit(doggedBuilder.buildAsRunnable(queue, new NoopWriter()));
- InMemCubeBuilderTest.feedData(cube, flatTable, queue, INPUT_ROWS, randSeed);
- future.get();
- }
- }
-
- class NoopWriter implements ICuboidWriter {
- @Override
- public void write(long cuboidId, GTRecord record) throws IOException {
- }
-
- @Override
- public void flush() {
-
- }
-
- @Override
- public void close() {
-
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/d2476f30/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 979224d..fe426a9 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -112,6 +112,13 @@
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-cube</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-storage</artifactId>
<type>test-jar</type>
<scope>test</scope>
@@ -282,7 +289,7 @@
<value>true</value>
</property>
</systemProperties>
- <argLine>-Xms1G -Xmx4G -XX:PermSize=128M -XX:MaxPermSize=512M</argLine>
+ <argLine>-Xms1G -Xmx2G -XX:PermSize=128M -XX:MaxPermSize=512M</argLine>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/kylin/blob/d2476f30/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
new file mode 100644
index 0000000..4291d91
--- /dev/null
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.inmemcubing;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.dimension.Dictionary;
+import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * very time consuming
+ */
+public class ITDoggedCubeBuilderStressTest extends LocalFileMetadataTestCase {
+
+ @SuppressWarnings("unused")
+ private static final Logger logger = LoggerFactory.getLogger(ITDoggedCubeBuilderStressTest.class);
+
+ // CI sandbox memory is no more than 512MB, this many input should hit memory threshold
+ private static final int INPUT_ROWS = 200000;
+ private static final int THREADS = 4;
+
+ private static CubeInstance cube;
+ private static String flatTable;
+ private static Map<TblColRef, Dictionary<String>> dictionaryMap;
+
+ @BeforeClass
+ public static void before() throws IOException {
+ staticCreateTestMetadata();
+
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
+
+ cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
+ flatTable = "../examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv";
+ dictionaryMap = InMemCubeBuilderTest.getDictionaryMap(cube, flatTable);
+ }
+
+ @AfterClass
+ public static void after() throws Exception {
+ staticCleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws Exception {
+
+ ArrayBlockingQueue<List<String>> queue = new ArrayBlockingQueue<List<String>>(1000);
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ long randSeed = System.currentTimeMillis();
+
+ DoggedCubeBuilder doggedBuilder = new DoggedCubeBuilder(cube.getDescriptor(), dictionaryMap);
+ doggedBuilder.setConcurrentThreads(THREADS);
+
+ {
+ Future<?> future = executorService.submit(doggedBuilder.buildAsRunnable(queue, new NoopWriter()));
+ InMemCubeBuilderTest.feedData(cube, flatTable, queue, INPUT_ROWS, randSeed);
+ future.get();
+ }
+ }
+
+ class NoopWriter implements ICuboidWriter {
+ @Override
+ public void write(long cuboidId, GTRecord record) throws IOException {
+ }
+
+ @Override
+ public void flush() {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/d2476f30/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7e521e1..593bbff 100644
--- a/pom.xml
+++ b/pom.xml
@@ -670,7 +670,7 @@
<value>false</value>
</property>
</systemProperties>
- <argLine>-Xmx4G -XX:MaxPermSize=512M</argLine>
+ <argLine>-Xms1G -Xmx2G -XX:MaxPermSize=512M</argLine>
</configuration>
</plugin>