You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by nj...@apache.org on 2016/04/18 08:35:25 UTC
[43/50] kylin git commit: KYLIN-1475 Inject ehcache manager for any
test case that will touch ehcache manager
KYLIN-1475 Inject ehcache manager for any test case that will touch ehcache manager
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83f49c34
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83f49c34
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83f49c34
Branch: refs/heads/1.4-rc
Commit: 83f49c34f0302bfcc323a760bf6097051006546f
Parents: e177ef5
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Mar 8 14:30:56 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Mar 8 14:30:56 2016 +0800
----------------------------------------------------------------------
.../dict/TupleFilterDictionaryTranslater.java | 166 -------------------
.../dict/TupleFilterFunctionTransformer.java | 161 ++++++++++++++++++
.../filter/ITupleFilterTransformer.java | 23 +++
.../metadata/filter/ITupleFilterTranslator.java | 26 ---
.../metadata/filter/function/BuiltInMethod.java | 24 ++-
.../cache/AbstractCacheFledgedQuery.java | 29 +---
.../kylin/storage/cache/DynamicCacheTest.java | 31 +++-
.../kylin/storage/cache/StaticCacheTest.java | 32 +++-
.../apache/kylin/query/ITKylinQueryTest.java | 15 ++
.../kylin/storage/hbase/ITStorageTest.java | 22 ++-
.../common/coprocessor/FilterDecorator.java | 6 +-
.../hbase/cube/v2/CubeSegmentScanner.java | 6 +-
12 files changed, 285 insertions(+), 256 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
deleted file mode 100644
index fec6a85..0000000
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import com.google.common.primitives.Primitives;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.FunctionTupleFilter;
-import org.apache.kylin.metadata.filter.ITupleFilterTranslator;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ListIterator;
-
-/**
- * Created by dongli on 1/7/16.
- */
-public class TupleFilterDictionaryTranslater implements ITupleFilterTranslator {
- public static final Logger logger = LoggerFactory.getLogger(TupleFilterDictionaryTranslater.class);
-
- private IDictionaryAware dictionaryAware;
-
- public TupleFilterDictionaryTranslater(IDictionaryAware dictionaryAware) {
- this.dictionaryAware = dictionaryAware;
- }
-
- @Override
- public TupleFilter translate(TupleFilter tupleFilter) {
- TupleFilter translated = null;
- if (tupleFilter instanceof CompareTupleFilter) {
- translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
- if (translated != null) {
- logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
- }
- } else if (tupleFilter instanceof FunctionTupleFilter) {
- translated = translateFunctionTupleFilter((FunctionTupleFilter) tupleFilter);
- if (translated != null) {
- logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
- }
- } else if (tupleFilter instanceof LogicalTupleFilter) {
- ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
- while (childIterator.hasNext()) {
- TupleFilter tempTranslated = translate(childIterator.next());
- if (tempTranslated != null)
- childIterator.set(tempTranslated);
- }
- }
- return translated == null ? tupleFilter : translated;
- }
-
- private TupleFilter translateFunctionTupleFilter(FunctionTupleFilter functionTupleFilter) {
- if (!functionTupleFilter.isValid())
- return null;
-
- TblColRef columnRef = functionTupleFilter.getColumn();
- Dictionary<?> dict = dictionaryAware.getDictionary(columnRef);
- if (dict == null)
- return null;
-
- CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
- translated.addChild(new ColumnTupleFilter(columnRef));
-
- try {
- for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
- Object dictVal = dict.getValueFromId(i);
- if ((Boolean) functionTupleFilter.invokeFunction(dictVal)) {
- translated.addChild(new ConstantTupleFilter(dictVal));
- }
- }
- } catch (Exception e) {
- logger.debug(e.getMessage());
- return null;
- }
- return translated;
- }
-
- @SuppressWarnings("unchecked")
- private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) {
- if (compTupleFilter.getFunction() == null)
- return null;
-
- FunctionTupleFilter functionTupleFilter = compTupleFilter.getFunction();
- if (!functionTupleFilter.isValid())
- return null;
-
- TblColRef columnRef = functionTupleFilter.getColumn();
- Dictionary<?> dict = dictionaryAware.getDictionary(columnRef);
- if (dict == null)
- return null;
-
- CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
- translated.addChild(new ColumnTupleFilter(columnRef));
-
- try {
- for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
- Object dictVal = dict.getValueFromId(i);
- Object computedVal = functionTupleFilter.invokeFunction(dictVal);
- Class clazz = Primitives.wrap(computedVal.getClass());
- Object targetVal = compTupleFilter.getFirstValue();
- if (Primitives.isWrapperType(clazz))
- targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue()));
-
- int comp = ((Comparable) computedVal).compareTo(targetVal);
- boolean compResult = false;
- switch (compTupleFilter.getOperator()) {
- case EQ:
- compResult = comp == 0;
- break;
- case NEQ:
- compResult = comp != 0;
- break;
- case LT:
- compResult = comp < 0;
- break;
- case LTE:
- compResult = comp <= 0;
- break;
- case GT:
- compResult = comp > 0;
- break;
- case GTE:
- compResult = comp >= 0;
- break;
- case IN:
- compResult = compTupleFilter.getValues().contains(computedVal.toString());
- break;
- case NOTIN:
- compResult = !compTupleFilter.getValues().contains(computedVal.toString());
- break;
- default:
- break;
- }
- if (compResult) {
- translated.addChild(new ConstantTupleFilter(dictVal));
- }
- }
- } catch (Exception e) {
- logger.debug(e.getMessage());
- return null;
- }
- return translated;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java
new file mode 100644
index 0000000..eadc291
--- /dev/null
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTransformer.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.dict;
+
+import com.google.common.primitives.Primitives;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.FunctionTupleFilter;
+import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ListIterator;
+
+public class TupleFilterFunctionTransformer implements ITupleFilterTransformer {
+ public static final Logger logger = LoggerFactory.getLogger(TupleFilterFunctionTransformer.class);
+
+ private IDictionaryAware dictionaryAware;
+
+ public TupleFilterFunctionTransformer(IDictionaryAware dictionaryAware) {
+ this.dictionaryAware = dictionaryAware;
+ }
+
+ @Override
+ public TupleFilter translate(TupleFilter tupleFilter) {
+ TupleFilter translated = null;
+ if (tupleFilter instanceof CompareTupleFilter) {
+ translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
+ if (translated != null) {
+ logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
+ }
+ } else if (tupleFilter instanceof FunctionTupleFilter) {
+ translated = translateFunctionTupleFilter((FunctionTupleFilter) tupleFilter);
+ if (translated != null) {
+ logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
+ }
+ } else if (tupleFilter instanceof LogicalTupleFilter) {
+ ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
+ while (childIterator.hasNext()) {
+ TupleFilter tempTranslated = translate(childIterator.next());
+ if (tempTranslated != null)
+ childIterator.set(tempTranslated);
+ }
+ }
+ return translated == null ? tupleFilter : translated;
+ }
+
+ private TupleFilter translateFunctionTupleFilter(FunctionTupleFilter functionTupleFilter) {
+ if (!functionTupleFilter.isValid())
+ return null;
+
+ TblColRef columnRef = functionTupleFilter.getColumn();
+ Dictionary<?> dict = dictionaryAware.getDictionary(columnRef);
+ if (dict == null)
+ return null;
+
+ CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
+ translated.addChild(new ColumnTupleFilter(columnRef));
+
+ try {
+ for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
+ Object dictVal = dict.getValueFromId(i);
+ if ((Boolean) functionTupleFilter.invokeFunction(dictVal)) {
+ translated.addChild(new ConstantTupleFilter(dictVal));
+ }
+ }
+ } catch (Exception e) {
+ logger.debug(e.getMessage());
+ return null;
+ }
+ return translated;
+ }
+
+ @SuppressWarnings("unchecked")
+ private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) {
+ if (compTupleFilter.getFunction() == null)
+ return null;
+
+ FunctionTupleFilter functionTupleFilter = compTupleFilter.getFunction();
+ if (!functionTupleFilter.isValid())
+ return null;
+
+ TblColRef columnRef = functionTupleFilter.getColumn();
+ Dictionary<?> dict = dictionaryAware.getDictionary(columnRef);
+ if (dict == null)
+ return null;
+
+ CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
+ translated.addChild(new ColumnTupleFilter(columnRef));
+
+ try {
+ for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
+ Object dictVal = dict.getValueFromId(i);
+ Object computedVal = functionTupleFilter.invokeFunction(dictVal);
+ Class clazz = Primitives.wrap(computedVal.getClass());
+ Object targetVal = compTupleFilter.getFirstValue();
+ if (Primitives.isWrapperType(clazz))
+ targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue()));
+
+ int comp = ((Comparable) computedVal).compareTo(targetVal);
+ boolean compResult = false;
+ switch (compTupleFilter.getOperator()) {
+ case EQ:
+ compResult = comp == 0;
+ break;
+ case NEQ:
+ compResult = comp != 0;
+ break;
+ case LT:
+ compResult = comp < 0;
+ break;
+ case LTE:
+ compResult = comp <= 0;
+ break;
+ case GT:
+ compResult = comp > 0;
+ break;
+ case GTE:
+ compResult = comp >= 0;
+ break;
+ case IN:
+ compResult = compTupleFilter.getValues().contains(computedVal.toString());
+ break;
+ case NOTIN:
+ compResult = !compTupleFilter.getValues().contains(computedVal.toString());
+ break;
+ default:
+ break;
+ }
+ if (compResult) {
+ translated.addChild(new ConstantTupleFilter(dictVal));
+ }
+ }
+ } catch (Exception e) {
+ logger.debug(e.getMessage());
+ return null;
+ }
+ return translated;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java
new file mode 100644
index 0000000..05fa7b0
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+public interface ITupleFilterTransformer {
+ TupleFilter translate(TupleFilter tupleFilter);
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java
deleted file mode 100644
index aed284c..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTranslator.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.filter;
-
-/**
- * Created by dongli on 1/7/16.
- */
-public interface ITupleFilterTranslator {
- TupleFilter translate(TupleFilter tupleFilter);
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
index b927d8d..97bca3b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
@@ -53,13 +53,17 @@ public enum BuiltInMethod {
this.method = MethodUtils.getMatchingAccessibleMethod(clazz, methodName, argumentTypes);
}
- /** SQL {@code LIKE} function. */
+ /**
+ * SQL {@code LIKE} function.
+ */
public static boolean like(String s, String pattern) {
final String regex = Like.sqlToRegexLike(pattern, null);
return Pattern.matches(regex, s);
}
- /** SQL INITCAP(string) function. */
+ /**
+ * SQL INITCAP(string) function.
+ */
public static String initcap(String s) {
// Assumes Alpha as [A-Za-z0-9]
// white space is treated as everything else.
@@ -96,22 +100,30 @@ public enum BuiltInMethod {
return newS.toString();
}
- /** SQL CHARACTER_LENGTH(string) function. */
+ /**
+ * SQL CHARACTER_LENGTH(string) function.
+ */
public static int charLength(String s) {
return s.length();
}
- /** SQL SUBSTRING(string FROM ... FOR ...) function. */
+ /**
+ * SQL SUBSTRING(string FROM ... FOR ...) function.
+ */
public static String substring(String s, int from, int for_) {
return s.substring(from - 1, Math.min(from - 1 + for_, s.length()));
}
- /** SQL UPPER(string) function. */
+ /**
+ * SQL UPPER(string) function.
+ */
public static String upper(String s) {
return s.toUpperCase();
}
- /** SQL LOWER(string) function. */
+ /**
+ * SQL LOWER(string) function.
+ */
public static String lower(String s) {
return s.toLowerCase();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
index 083021e..7826c0c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
@@ -55,31 +55,6 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup
CACHE_MANAGER = cacheManager;
}
- /**
- * This method is only useful non-spring injected test cases.
- * When Kylin is normally ran as a spring app CACHE_MANAGER will be injected.
- * and the configuration for cache lies in server/src/main/resources/ehcache.xml
- *
- * the cache named "StorageCache" acts like a template for each realization to
- * create its own cache.
- */
- private static void initCacheManger() {
- Configuration conf = new Configuration();
- conf.setMaxBytesLocalHeap("128M");
- CACHE_MANAGER = CacheManager.create(conf);
-
- //a fake template for test cases
- Cache storageCache = new Cache(new CacheConfiguration(storageCacheTemplate, 0).//
- memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU).//
- eternal(false).//
- timeToIdleSeconds(86400).//
- diskExpiryThreadIntervalSeconds(0).//
- //maxBytesLocalHeap(10, MemoryUnit.MEGABYTES).//
- persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.NONE)));
-
- CACHE_MANAGER.addCacheIfAbsent(storageCache);
- }
-
protected StreamSQLResult getStreamSQLResult(StreamSQLDigest streamSQLDigest) {
Cache cache = CACHE_MANAGER.getCache(this.underlyingStorage.getStorageUUID());
@@ -105,10 +80,10 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup
private void makeCacheIfNecessary(String storageUUID) {
if (CACHE_MANAGER == null || (!(CACHE_MANAGER.getStatus().equals(Status.STATUS_ALIVE)))) {
- logger.warn("CACHE_MANAGER is not provided or not alive");
- initCacheManger();
+ throw new RuntimeException("CACHE_MANAGER is not provided or not alive");
}
+
if (CACHE_MANAGER.getCache(storageUUID) == null) {
logger.info("Cache for {} initializing...", storageUUID);
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
index c9ea479..732f5f6 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
@@ -18,11 +18,10 @@
package org.apache.kylin.storage.cache;
-import java.util.ArrayList;
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
+import com.google.common.collect.Ranges;
+import net.sf.ehcache.CacheManager;
import org.apache.commons.lang.NotImplementedException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.DateFormat;
@@ -38,22 +37,36 @@ import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.tuple.Tuple;
import org.apache.kylin.storage.tuple.TupleInfo;
+import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
-import com.google.common.collect.Ranges;
+import java.util.ArrayList;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
/**
*/
public class DynamicCacheTest {
+ private static CacheManager cacheManager;
+
+
@BeforeClass
public static void setup() {
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cache.threshold.duration", "0");
+
+ cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml");
+ AbstractCacheFledgedQuery.setCacheManager(cacheManager);
+ }
+
+ @AfterClass
+ public static void tearDownResource() {
+ cacheManager.shutdown();
+ AbstractCacheFledgedQuery.setCacheManager(null);
}
@@ -105,7 +118,7 @@ public class DynamicCacheTest {
final List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
final TupleInfo tupleInfo = StorageMockUtils.newTupleInfo(groups, aggregations);
- SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", null, null, Lists.<TblColRef> newArrayList(), groups, Lists.newArrayList(partitionCol), Lists.<TblColRef> newArrayList(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>());
+ SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", null, null, Lists.<TblColRef>newArrayList(), groups, Lists.newArrayList(partitionCol), Lists.<TblColRef>newArrayList(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>());
ITuple aTuple = new TsOnlyTuple(partitionCol, "2011-02-01");
ITuple bTuple = new TsOnlyTuple(partitionCol, "2012-02-01");
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
index 3b50736..bdbb378 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
@@ -18,12 +18,9 @@
package org.apache.kylin.storage.cache;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
+import net.sf.ehcache.CacheManager;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.IdentityUtils;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -38,22 +35,39 @@ import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.tuple.Tuple;
import org.apache.kylin.storage.tuple.TupleInfo;
+import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
/**
*/
public class StaticCacheTest {
+ private static CacheManager cacheManager;
+
+
@BeforeClass
public static void setup() {
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cache.threshold.duration", "0");
+
+ cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml");
+ AbstractCacheFledgedQuery.setCacheManager(cacheManager);
}
+ @AfterClass
+ public static void tearDownResource() {
+ cacheManager.shutdown();
+ AbstractCacheFledgedQuery.setCacheManager(null);
+ }
+
+
@Test
public void basicTest() {
@@ -61,7 +75,7 @@ public class StaticCacheTest {
final List<TblColRef> groups = StorageMockUtils.buildGroups();
final List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
final TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0));
- final SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>());
+ final SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef>emptySet(), groups, Collections.<TblColRef>emptySet(), Collections.<TblColRef>emptySet(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>());
final TupleInfo tupleInfo = StorageMockUtils.newTupleInfo(groups, aggregations);
final List<ITuple> ret = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index d7dd260..c400929 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -25,12 +25,14 @@ import java.sql.DriverManager;
import java.util.List;
import java.util.Properties;
+import net.sf.ehcache.CacheManager;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
+import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.dbunit.database.DatabaseConnection;
@@ -42,6 +44,8 @@ import org.junit.Test;
@Ignore("KylinQueryTest is contained by ITCombinationTest")
public class ITKylinQueryTest extends KylinTestBase {
+ private static CacheManager cacheManager;
+
@BeforeClass
public static void setUp() throws Exception {
@@ -74,6 +78,11 @@ public class ITKylinQueryTest extends KylinTestBase {
// Load H2 Tables (inner join)
H2Database h2DB = new H2Database(h2Connection, config);
h2DB.loadAllTables();
+
+ cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml");
+ AbstractCacheFledgedQuery.setCacheManager(cacheManager);
+
+
}
protected static void clean() {
@@ -84,6 +93,12 @@ public class ITKylinQueryTest extends KylinTestBase {
ObserverEnabler.forceCoprocessorUnset();
HBaseMetadataTestCase.staticCleanupTestMetadata();
+
+ if (cacheManager != null) {
+ cacheManager.shutdown();
+ }
+ AbstractCacheFledgedQuery.setCacheManager(null);
+
}
@Ignore("this is only for debug")
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
index 5ce26c7..5a24d28 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
@@ -18,12 +18,7 @@
package org.apache.kylin.storage.hbase;
-import static org.junit.Assert.*;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
+import net.sf.ehcache.CacheManager;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
@@ -38,6 +33,7 @@ import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.storage.IStorageQuery;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.StorageFactory;
+import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery;
import org.apache.kylin.storage.cache.StorageMockUtils;
import org.apache.kylin.storage.exception.ScanOutOfLimitException;
import org.junit.After;
@@ -48,7 +44,15 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+
public class ITStorageTest extends HBaseMetadataTestCase {
+ private static CacheManager cacheManager;
+
private IStorageQuery storageEngine;
private CubeInstance cube;
@@ -56,10 +60,14 @@ public class ITStorageTest extends HBaseMetadataTestCase {
@BeforeClass
public static void setupResource() throws Exception {
+ cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml");
+ AbstractCacheFledgedQuery.setCacheManager(cacheManager);
}
@AfterClass
public static void tearDownResource() {
+ cacheManager.shutdown();
+ AbstractCacheFledgedQuery.setCacheManager(null);
}
@Before
@@ -144,7 +152,7 @@ public class ITStorageTest extends HBaseMetadataTestCase {
int count = 0;
ITupleIterator iterator = null;
try {
- SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>());
+ SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef>emptySet(), groups, Collections.<TblColRef>emptySet(), Collections.<TblColRef>emptySet(), aggregations, new ArrayList<MeasureDesc>(), new ArrayList<SQLDigest.OrderEnum>());
iterator = storageEngine.search(context, sqlDigest, StorageMockUtils.newTupleInfo(groups, aggregations));
while (iterator.hasNext()) {
ITuple tuple = iterator.next();
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
index cb73b22..ab6c8b8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
@@ -26,11 +26,11 @@ import org.apache.kylin.cube.kv.RowKeyColumnIO;
import org.apache.kylin.dict.DictCodeSystem;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.kylin.dict.TupleFilterDictionaryTranslater;
+import org.apache.kylin.dict.TupleFilterFunctionTransformer;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.ITupleFilterTranslator;
+import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.model.TblColRef;
@@ -149,7 +149,7 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator {
if (filter == null)
return null;
- ITupleFilterTranslator translator = new TupleFilterDictionaryTranslater(columnIO.getIDictionaryAware());
+ ITupleFilterTransformer translator = new TupleFilterFunctionTransformer(columnIO.getIDictionaryAware());
filter = translator.translate(filter);
// un-evaluatable filter is replaced with TRUE
http://git-wip-us.apache.org/repos/asf/kylin/blob/83f49c34/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index 6f3d012..55b9b15 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -41,7 +41,7 @@ import org.apache.kylin.cube.gridtable.CubeGridTable;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.dict.TupleFilterDictionaryTranslater;
+import org.apache.kylin.dict.TupleFilterFunctionTransformer;
import org.apache.kylin.gridtable.EmptyGTScanner;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
@@ -50,7 +50,7 @@ import org.apache.kylin.gridtable.GTScanRangePlanner;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTUtil;
import org.apache.kylin.gridtable.IGTScanner;
-import org.apache.kylin.metadata.filter.ITupleFilterTranslator;
+import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -82,7 +82,7 @@ public class CubeSegmentScanner implements IGTScanner {
CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
// translate FunctionTupleFilter to IN clause
- ITupleFilterTranslator translator = new TupleFilterDictionaryTranslater(this.cubeSeg);
+ ITupleFilterTransformer translator = new TupleFilterFunctionTransformer(this.cubeSeg);
filter = translator.translate(filter);
//replace the constant values in filter to dictionary codes