You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/04/26 06:27:14 UTC
[32/50] incubator-kylin git commit: KYLIN-671 renaming
KYLIN-671 renaming
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/92889025
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/92889025
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/92889025
Branch: refs/heads/streaming-localdict
Commit: 92889025f5369ad96dfc7f8334bbdb2d3869831d
Parents: 48a8779
Author: honma <ho...@ebay.com>
Authored: Wed Apr 15 16:15:52 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Apr 22 16:45:38 2015 +0800
----------------------------------------------------------------------
.../kylin/storage/StorageEngineFactory.java | 4 +-
.../cache/CacheFledgedStorageEngine.java | 109 +++++++++++++++++++
.../cache/CacheFledgedTupleIterator.java | 109 -------------------
.../kylin/storage/cache/StreamSQLResult.java | 8 +-
4 files changed, 115 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/92889025/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java b/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
index df3371f..1a36bc4 100644
--- a/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
+++ b/storage/src/main/java/org/apache/kylin/storage/StorageEngineFactory.java
@@ -22,7 +22,7 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.storage.cache.CacheFledgedTupleIterator;
+import org.apache.kylin.storage.cache.CacheFledgedStorageEngine;
import org.apache.kylin.storage.hbase.CubeStorageEngine;
import org.apache.kylin.storage.hbase.InvertedIndexStorageEngine;
import org.apache.kylin.storage.hybrid.HybridInstance;
@@ -35,7 +35,7 @@ public class StorageEngineFactory {
public static IStorageEngine getStorageEngine(IRealization realization, boolean allowCache) {
if (allowCache) {
- return new CacheFledgedTupleIterator(realization);
+ return new CacheFledgedStorageEngine(realization);
}
if (realization.getType() == RealizationType.INVERTED_INDEX) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/92889025/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStorageEngine.java
new file mode 100644
index 0000000..b1fe17f
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStorageEngine.java
@@ -0,0 +1,109 @@
+package org.apache.kylin.storage.cache;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.config.CacheConfiguration;
+import net.sf.ehcache.config.MemoryUnit;
+import net.sf.ehcache.config.PersistenceConfiguration;
+import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.RangeUtil;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.*;
+import org.apache.kylin.metadata.tuple.CompoundTupleIterator;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
+import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.StorageEngineFactory;
+import org.apache.kylin.storage.hbase.coprocessor.endpoint.TsConditionExtractor;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Range;
+
+import javax.annotation.Nullable;
+import java.util.List;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 4/13/15.
+ */
+public class CacheFledgedStorageEngine implements IStorageEngine {
+
+ public static final String SUCCESS_QUERY_CACHE = "SuccessQueryCache";
+ public static final String EXCEPTION_QUERY_CACHE = "ExceptionQueryCache";//TODO
+
+ static CacheManager cacheManager;
+
+ static {
+ cacheManager = CacheManager.create();
+
+ //Create a Cache specifying its configuration.
+ Cache successCache = new Cache(new CacheConfiguration(SUCCESS_QUERY_CACHE, 0).//
+ memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LFU).//
+ eternal(false).//
+ timeToIdleSeconds(86400).//
+ diskExpiryThreadIntervalSeconds(0).//
+ maxBytesLocalHeap(500, MemoryUnit.MEGABYTES).//
+ persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.LOCALTEMPSWAP)));
+
+ cacheManager.addCache(successCache);
+ }
+
+ private TblColRef partitionColRef;
+ private IRealization realization;
+
+ public CacheFledgedStorageEngine(IRealization realization) {
+ this.realization = realization;
+
+ Preconditions.checkArgument(realization.getType() != RealizationType.CUBE, "Cube realization does not need dynamic cache!");
+ String modelName = realization.getModelName();
+ DataModelDesc dataModelDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getDataModelDesc(modelName);
+ PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
+ Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
+ assert partitionDesc != null;
+ partitionColRef = partitionDesc.getPartitionDateColumnRef();
+ Preconditions.checkArgument(partitionColRef != null, "getPartitionDateColumnRef for " + realization + " is null");
+ }
+
+ @Override
+ public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest) {
+ StreamSQLDigest streamSQLDigest = new StreamSQLDigest(sqlDigest, partitionColRef);
+ StreamSQLResult cachedResult = (StreamSQLResult) cacheManager.getCache(SUCCESS_QUERY_CACHE).get(streamSQLDigest).getObjectValue();
+
+ ITupleIterator ret;
+
+ if (cachedResult != null) {
+ Range<Long> tsRange = TsConditionExtractor.extractTsCondition(partitionColRef, sqlDigest.filter);
+ Range<Long> reusePeriod = cachedResult.getReusableResults(tsRange);
+ if (reusePeriod != null) {
+ List<Range<Long>> remainings = RangeUtil.remove(tsRange, reusePeriod);
+ if (remainings.size() == 1) {
+ SimpleTupleIterator reusedTuples = new SimpleTupleIterator(cachedResult.reuse(reusePeriod));
+ Range<Long> remaining = remainings.get(0);
+ ITupleIterator freshTuples = SQLDigestUtil.appendTsFilterToExecute(sqlDigest, partitionColRef, remaining, new Function<Void, ITupleIterator>() {
+ @Nullable
+ @Override
+ public ITupleIterator apply(Void input) {
+ return StorageEngineFactory.getStorageEngine(realization, false).search(context, sqlDigest);
+ }
+ });
+ ret = new CompoundTupleIterator(Lists.newArrayList(reusedTuples, freshTuples));
+ //TODO:update cache
+ return ret;//cache successfully reused
+ }
+ }
+ }
+
+ //cache cannot reuse case:
+ ret = StorageEngineFactory.getStorageEngine(realization, false).search(context, sqlDigest);
+ //TODO:update cache
+ return ret;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/92889025/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedTupleIterator.java
deleted file mode 100644
index ab7ab3b..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedTupleIterator.java
+++ /dev/null
@@ -1,109 +0,0 @@
-package org.apache.kylin.storage.cache;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheManager;
-import net.sf.ehcache.config.CacheConfiguration;
-import net.sf.ehcache.config.MemoryUnit;
-import net.sf.ehcache.config.PersistenceConfiguration;
-import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.RangeUtil;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.PartitionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.*;
-import org.apache.kylin.metadata.tuple.CompoundTupleIterator;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
-import org.apache.kylin.storage.IStorageEngine;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.StorageEngineFactory;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.TsConditionExtractor;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Range;
-
-import javax.annotation.Nullable;
-import java.util.List;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 4/13/15.
- */
-public class CacheFledgedTupleIterator implements IStorageEngine {
-
- public static final String SUCCESS_QUERY_CACHE = "SuccessQueryCache";
- public static final String EXCEPTION_QUERY_CACHE = "ExceptionQueryCache";//TODO
-
- static CacheManager cacheManager;
-
- static {
- cacheManager = CacheManager.create();
-
- //Create a Cache specifying its configuration.
- Cache successCache = new Cache(new CacheConfiguration(SUCCESS_QUERY_CACHE, 0).//
- memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LFU).//
- eternal(false).//
- timeToIdleSeconds(86400).//
- diskExpiryThreadIntervalSeconds(0).//
- maxBytesLocalHeap(500, MemoryUnit.MEGABYTES).//
- persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.LOCALTEMPSWAP)));
-
- cacheManager.addCache(successCache);
- }
-
- private TblColRef partitionColRef;
- private IRealization realization;
-
- public CacheFledgedTupleIterator(IRealization realization) {
- this.realization = realization;
-
- Preconditions.checkArgument(realization.getType() != RealizationType.CUBE, "Cube realization does not need dynamic cache!");
- String modelName = realization.getModelName();
- DataModelDesc dataModelDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getDataModelDesc(modelName);
- PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
- Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
- assert partitionDesc != null;
- partitionColRef = partitionDesc.getPartitionDateColumnRef();
- Preconditions.checkArgument(partitionColRef != null, "getPartitionDateColumnRef for " + realization + " is null");
- }
-
- @Override
- public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest) {
- StreamSQLDigest streamSQLDigest = new StreamSQLDigest(sqlDigest, partitionColRef);
- StreamSQLResult cachedResult = (StreamSQLResult) cacheManager.getCache(SUCCESS_QUERY_CACHE).get(streamSQLDigest).getObjectValue();
-
- ITupleIterator ret;
-
- if (cachedResult != null) {
- Range<Long> tsRange = TsConditionExtractor.extractTsCondition(partitionColRef, sqlDigest.filter);
- Range<Long> reusePeriod = cachedResult.getReusableResults(tsRange);
- if (reusePeriod != null) {
- List<Range<Long>> remainings = RangeUtil.remove(tsRange, reusePeriod);
- if (remainings.size() == 1) {
- SimpleTupleIterator reusedTuples = new SimpleTupleIterator(cachedResult.reuse(reusePeriod));
- Range<Long> remaining = remainings.get(0);
- ITupleIterator freshTuples = SQLDigestUtil.appendTsFilterToExecute(sqlDigest, partitionColRef, remaining, new Function<Void, ITupleIterator>() {
- @Nullable
- @Override
- public ITupleIterator apply(Void input) {
- return StorageEngineFactory.getStorageEngine(realization, false).search(context, sqlDigest);
- }
- });
- ret = new CompoundTupleIterator(Lists.newArrayList(reusedTuples, freshTuples));
- //TODO:update cache
- return ret;//cache successfully reused
- }
- }
- }
-
- //cache cannot reuse case:
- ret = StorageEngineFactory.getStorageEngine(realization, false).search(context, sqlDigest);
- //TODO:update cache
- return ret;
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/92889025/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java b/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
index b30db25..f537fc7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
@@ -1,12 +1,12 @@
package org.apache.kylin.storage.cache;
-import com.google.common.collect.Range;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.storage.tuple.Tuple;
-
import java.util.Iterator;
import java.util.List;
+import org.apache.kylin.metadata.tuple.ITuple;
+
+import com.google.common.collect.Range;
+
/**
* Created by Hongbin Ma(Binmahone) on 4/13/15.
*/