You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/11/16 09:03:46 UTC
[1/2] incubator-kylin git commit: KYLIN-1142 fix ut
Repository: incubator-kylin
Updated Branches:
refs/heads/2.x-staging 0273a145f -> b9a19110b
KYLIN-1142 fix ut
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/91de85c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/91de85c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/91de85c1
Branch: refs/heads/2.x-staging
Commit: 91de85c108adad845abf8629b0c7a7cdb7dae365
Parents: 0273a14
Author: honma <ho...@ebay.com>
Authored: Mon Nov 16 16:05:20 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Nov 16 16:05:20 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/jdbc/json/SQLResponseStub.java | 21 +++++++++++++++-----
1 file changed, 16 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/91de85c1/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java b/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
index f34e48c..365b848 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
@@ -51,7 +51,10 @@ public class SQLResponseStub implements Serializable {
private long totalScanCount;
- private boolean hitCache = false;
+
+ private boolean hitExceptionCache = false;
+
+ private boolean storageCacheUsed = false;
public SQLResponseStub() {
}
@@ -128,12 +131,20 @@ public class SQLResponseStub implements Serializable {
this.totalScanCount = totalScanCount;
}
- public boolean isHitCache() {
- return hitCache;
+ public boolean isHitExceptionCache() {
+ return hitExceptionCache;
+ }
+
+ public void setHitExceptionCache(boolean hitExceptionCache) {
+ this.hitExceptionCache = hitExceptionCache;
+ }
+
+ public boolean isStorageCacheUsed() {
+ return storageCacheUsed;
}
- public void setHitCache(boolean hitCache) {
- this.hitCache = hitCache;
+ public void setStorageCacheUsed(boolean storageCacheUsed) {
+ this.storageCacheUsed = storageCacheUsed;
}
public static long getSerialversionuid() {
[2/2] incubator-kylin git commit: KYLIN-1147 use pooled size for each
cache(representing each realization)
Posted by ma...@apache.org.
KYLIN-1147 use pooled size for each cache(representing each realization)
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b9a19110
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b9a19110
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b9a19110
Branch: refs/heads/2.x-staging
Commit: b9a19110b826415faddf69d76dc69c2e25e07705
Parents: 91de85c
Author: honma <ho...@ebay.com>
Authored: Mon Nov 16 16:08:06 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Nov 16 16:08:06 2015 +0800
----------------------------------------------------------------------
.../cache/AbstractCacheFledgedQuery.java | 28 ++---
server/src/main/resources/ehcache-test.xml | 3 -
server/src/main/resources/ehcache.xml | 3 -
.../org/apache/kylin/rest/StorageCacheTest.java | 116 +++++++++++++++++++
4 files changed, 131 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
index a7bdae0..745ddd1 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
@@ -5,7 +5,6 @@ import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
import net.sf.ehcache.config.CacheConfiguration;
import net.sf.ehcache.config.Configuration;
-import net.sf.ehcache.config.MemoryUnit;
import net.sf.ehcache.config.PersistenceConfiguration;
import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
@@ -37,6 +36,14 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup
CACHE_MANAGER = cacheManager;
}
+ /**
+ * This method is only useful non-spring injected test cases.
+ * When Kylin is normally ran as a spring app CACHE_MANAGER will be injected.
+ * and the configuration for cache lies in server/src/main/resources/ehcache.xml
+ *
+ * the cache named "StorageCache" acts like a template for each realization to
+ * create its own cache.
+ */
private static void initCacheManger() {
Configuration conf = new Configuration();
conf.setMaxBytesLocalHeap("128M");
@@ -48,7 +55,7 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup
eternal(false).//
timeToIdleSeconds(86400).//
diskExpiryThreadIntervalSeconds(0).//
- maxBytesLocalHeap(10, MemoryUnit.MEGABYTES).//
+ //maxBytesLocalHeap(10, MemoryUnit.MEGABYTES).//
persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.NONE)));
CACHE_MANAGER.addCache(storageCache);
@@ -84,25 +91,20 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup
}
if (CACHE_MANAGER.getCache(storageUUID) == null) {
- logger.info("Cache for {} initting...", storageUUID);
+ logger.info("Cache for {} initializing...", storageUUID);
//Create a Cache specifying its configuration.
CacheConfiguration templateConf = CACHE_MANAGER.getCache(storageCacheTemplate).getCacheConfiguration();
- PersistenceConfiguration pconf = templateConf.getPersistenceConfiguration();
- if (pconf != null) {
- logger.info("PersistenceConfiguration strategy: " + pconf.getStrategy());
- } else {
- logger.warn("PersistenceConfiguration is null");
- }
-
- Cache storageCache = new Cache(new CacheConfiguration(storageUUID, (int) templateConf.getMaxEntriesLocalHeap()).//
+
+ Cache storageCache = new Cache(new CacheConfiguration(storageUUID, 0).//
memoryStoreEvictionPolicy(templateConf.getMemoryStoreEvictionPolicy()).//
eternal(templateConf.isEternal()).//
timeToIdleSeconds(templateConf.getTimeToIdleSeconds()).//
- maxBytesLocalHeap(templateConf.getMaxBytesLocalHeap(), MemoryUnit.BYTES).persistence(pconf));
- //TODO: deal with failed queries, and only cache too long query
+ //maxBytesLocalHeap(templateConf.getMaxBytesLocalHeap(), MemoryUnit.BYTES).//using pooled size
+ persistence(templateConf.getPersistenceConfiguration()));
CACHE_MANAGER.addCache(storageCache);
+
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/server/src/main/resources/ehcache-test.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/ehcache-test.xml b/server/src/main/resources/ehcache-test.xml
index 8ee2246..7cae6ce 100644
--- a/server/src/main/resources/ehcache-test.xml
+++ b/server/src/main/resources/ehcache-test.xml
@@ -3,7 +3,6 @@
eternal="false"
timeToIdleSeconds="86400"
memoryStoreEvictionPolicy="LRU"
- maxBytesLocalHeap="1M"
>
<persistence strategy="none"/>
</cache>
@@ -11,7 +10,6 @@
eternal="false"
timeToIdleSeconds="86400"
memoryStoreEvictionPolicy="LRU"
- maxBytesLocalHeap="1M"
>
<persistence strategy="none"/>
</cache>
@@ -19,7 +17,6 @@
eternal="false"
timeToLiveSeconds="10800"
memoryStoreEvictionPolicy="LRU"
- maxBytesLocalHeap="1M"
>
<persistence strategy="none"/>
</cache>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/server/src/main/resources/ehcache.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/ehcache.xml b/server/src/main/resources/ehcache.xml
index 29fce05..3f39702 100644
--- a/server/src/main/resources/ehcache.xml
+++ b/server/src/main/resources/ehcache.xml
@@ -3,7 +3,6 @@
eternal="false"
timeToIdleSeconds="86400"
memoryStoreEvictionPolicy="LRU"
- maxBytesLocalHeap="50M"
>
<persistence strategy="none"/>
</cache>
@@ -11,7 +10,6 @@
eternal="false"
timeToIdleSeconds="86400"
memoryStoreEvictionPolicy="LRU"
- maxBytesLocalHeap="50M"
>
<persistence strategy="none"/>
</cache>
@@ -19,7 +17,6 @@
eternal="false"
timeToLiveSeconds="10800"
memoryStoreEvictionPolicy="LRU"
- maxBytesLocalHeap="50M"
>
<persistence strategy="none"/>
</cache>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b9a19110/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java b/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java
new file mode 100644
index 0000000..e781e00
--- /dev/null
+++ b/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest;
+
+import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.Element;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.rest.service.ServiceTestBase;
+import org.apache.kylin.storage.ICachableStorageQuery;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery;
+import org.apache.kylin.storage.cache.CacheFledgedStaticQuery;
+import org.apache.kylin.storage.tuple.TupleInfo;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.google.common.collect.Range;
+
+public class StorageCacheTest extends ServiceTestBase {
+
+ public class MockedCachableStorageQuery implements ICachableStorageQuery {
+ private String uuid;
+
+ public MockedCachableStorageQuery(String uuid) {
+ this.uuid = uuid;
+ }
+
+ @Override
+ public boolean isDynamic() {
+ return false;
+ }
+
+ @Override
+ public Range<Long> getVolatilePeriod() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getStorageUUID() {
+ return this.uuid;
+ }
+
+ @Override
+ public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ throw new NotImplementedException();
+ }
+ }
+
+ @Autowired
+ private CacheManager cacheManager;
+
+ @Before
+ public void setup() throws Exception {
+ AbstractCacheFledgedQuery.setCacheManager(cacheManager);
+ new CacheFledgedStaticQuery(new MockedCachableStorageQuery("1"));
+ new CacheFledgedStaticQuery(new MockedCachableStorageQuery("2"));
+ new CacheFledgedStaticQuery(new MockedCachableStorageQuery("3"));
+ new CacheFledgedStaticQuery(new MockedCachableStorageQuery("4"));
+ new CacheFledgedStaticQuery(new MockedCachableStorageQuery("5"));
+ new CacheFledgedStaticQuery(new MockedCachableStorageQuery("6"));
+ new CacheFledgedStaticQuery(new MockedCachableStorageQuery("7"));
+ new CacheFledgedStaticQuery(new MockedCachableStorageQuery("8"));
+ new CacheFledgedStaticQuery(new MockedCachableStorageQuery("9"));
+ new CacheFledgedStaticQuery(new MockedCachableStorageQuery("10"));
+ new CacheFledgedStaticQuery(new MockedCachableStorageQuery("11"));
+ }
+
+ @Test
+ public void test1() {
+ int oneM = 1 << 20;
+ cacheManager.getCache("1").put(new Element("xx", new byte[oneM]));
+ Element xx = cacheManager.getCache("1").get("xx");
+ Assert.assertEquals(oneM, ((byte[]) xx.getObjectValue()).length);
+
+ cacheManager.getCache("2").put(new Element("yy", new byte[3 * oneM]));
+ Element yy = cacheManager.getCache("2").get("yy");
+ Assert.assertEquals(3 * oneM, ((byte[]) yy.getObjectValue()).length);
+
+ cacheManager.getCache("3").put(new Element("zz", new byte[10 * oneM]));
+ Element zz = cacheManager.getCache("3").get("zz");
+ Assert.assertEquals(null, zz);
+
+ cacheManager.getCache("4").put(new Element("aa", new byte[oneM]));
+ Element aa = cacheManager.getCache("4").get("aa");
+ Assert.assertEquals(oneM, ((byte[]) aa.getObjectValue()).length);
+
+ cacheManager.getCache("2").put(new Element("bb", new byte[3 * oneM]));
+ Element bb = cacheManager.getCache("2").get("bb");
+ Assert.assertEquals(3 * oneM, ((byte[]) bb.getObjectValue()).length);
+ }
+
+ @Test
+ public void test2() {
+ }
+}