You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/04/26 06:27:11 UTC
[29/50] incubator-kylin git commit: KYLIN-671 POC
KYLIN-671 POC
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a0e9fbe8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a0e9fbe8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a0e9fbe8
Branch: refs/heads/streaming-localdict
Commit: a0e9fbe81ef1e84389ad68390b17643da7899e1f
Parents: cb2c5b1
Author: honma <ho...@ebay.com>
Authored: Tue Apr 14 13:31:45 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Apr 22 16:44:51 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/BasicTest.java | 22 ++-
.../org/apache/kylin/common/util/RangeUtil.java | 17 +++
.../org/apache/kylin/cube/CubeInstance.java | 9 +-
.../apache/kylin/invertedindex/IIInstance.java | 5 +
.../metadata/filter/IgnoreTsCondition.java | 64 +++++++++
.../metadata/realization/IRealization.java | 2 +
.../kylin/metadata/realization/SQLDigest.java | 1 +
.../metadata/realization/StreamSQLDigest.java | 83 ++++++++++++
storage/pom.xml | 5 +
.../cache/CacheFledgedTupleIterator.java | 71 ++++++++++
.../kylin/storage/cache/StreamSQLResult.java | 13 ++
.../hbase/coprocessor/CoprocessorFilter.java | 12 +-
.../hbase/coprocessor/DictCodeSystem.java | 6 +-
.../hbase/coprocessor/FilterDecorator.java | 16 +--
.../endpoint/EndpointTupleIterator.java | 2 +-
.../kylin/storage/hybrid/HybridInstance.java | 1 +
.../apache/kylin/storage/cache/EhcacheTest.java | 46 +++++++
.../storage/filter/IgnoreTsConditionTest.java | 133 +++++++++++++++++++
.../endpoint/TsConditionExtractorTest.java | 8 +-
19 files changed, 487 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index bbe6e8b..a949599 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -24,8 +24,14 @@ import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.*;
+import com.google.common.base.Function;
import com.google.common.collect.Lists;
+
+import com.google.common.collect.Ordering;
+import com.google.common.collect.Range;
+import com.google.common.collect.Ranges;
import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.lang.math.IntRange;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Ignore;
import org.junit.Test;
@@ -62,15 +68,22 @@ public class BasicTest {
@Test
@Ignore("convenient trial tool for dev")
public void test1() throws Exception {
+
+
+ System.out.println(Ranges.open(3, 5).isConnected(Ranges.open(4, 10)));
+ System.out.println(Ranges.open(4, 10).isConnected(Ranges.open(3,5)));
+
+
String bb = "\\x00\\x00\\x00\\x00\\x01\\x3F\\xD0\\x2D\\58\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00";//2013/07/12 07:59:37
String cc = "\\x00\\x00\\x00\\x00\\x01\\x41\\xBE\\x8F\\xD8\\x00\\x00\\x00\\x00\\x00\\x00\\x00";//2013/10/16 08:00:00
String dd = "\\x00\\x00\\x00\\x00\\x01\\x41\\xBE\\x8F\\xD8\\x07\\x00\\x18\\x00\\x00\\x00";
+
byte[] bytes = BytesUtil.fromReadableText(dd);
- long ttt = BytesUtil.readLong(bytes,2,8);
+ long ttt = BytesUtil.readLong(bytes, 2, 8);
System.out.println(time(ttt));
System.out.println("\\");
- System.out.println("\\\\");
+ System.out.println("n");
System.out.println("The start key is set to " + null);
System.out.println(time(946684800000L));
@@ -99,9 +112,8 @@ public class BasicTest {
@Ignore("fix it later")
public void test2() throws IOException, ConfigurationException {
ArrayList<String> x = Lists.newArrayListWithCapacity(10);
- x.set(2,"dd");
- for(String y : x)
- {
+ x.set(2, "dd");
+ for (String y : x) {
System.out.println(y);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/common/src/test/java/org/apache/kylin/common/util/RangeUtil.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/RangeUtil.java b/common/src/test/java/org/apache/kylin/common/util/RangeUtil.java
new file mode 100644
index 0000000..59c7349
--- /dev/null
+++ b/common/src/test/java/org/apache/kylin/common/util/RangeUtil.java
@@ -0,0 +1,17 @@
+package org.apache.kylin.common.util;
+
+import com.google.common.collect.Range;
+import com.google.common.collect.Ranges;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 4/14/15.
+ */
+public class RangeUtil {
+ public static <C extends Comparable<?>> Range<C> retain(Range<C> self, Range<C> other) {
+ if (!self.isConnected(other) || self.intersection(other).isEmpty()) {
+ return null;
+ }
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index fa9244a..e63f836 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -89,7 +89,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
private static final int COST_WEIGHT_MEASURE = 1;
private static final int COST_WEIGHT_LOOKUP_TABLE = 1;
private static final int COST_WEIGHT_INNER_JOIN = 2;
-
+
public List<CubeSegment> getBuildingSegments() {
List<CubeSegment> buildingSegments = new ArrayList<CubeSegment>();
if (null != segments) {
@@ -335,7 +335,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
public int getCost(SQLDigest digest) {
int calculatedCost = cost;
- calculatedCost += getAllDimensions().size()* COST_WEIGHT_DIMENSION + getMeasures().size() * COST_WEIGHT_MEASURE;
+ calculatedCost += getAllDimensions().size() * COST_WEIGHT_DIMENSION + getMeasures().size() * COST_WEIGHT_MEASURE;
for (LookupDesc lookupDesc : this.getDescriptor().getModel().getLookups()) {
// more tables, more cost
@@ -394,6 +394,11 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
}
@Override
+ public String getModelName() {
+ return this.getDescriptor().getModelName();
+ }
+
+ @Override
public List<TblColRef> getAllDimensions() {
return Lists.newArrayList(getDescriptor().listDimensionColumnsIncludingDerived());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
index fe0d2de..1d300a8 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
@@ -351,6 +351,11 @@ public class IIInstance extends RootPersistentEntity implements IRealization {
}
@Override
+ public String getModelName() {
+ return this.getDescriptor().getModelName();
+ }
+
+ @Override
public List<TblColRef> getAllDimensions() {
return getDescriptor().listAllDimensions();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/metadata/src/main/java/org/apache/kylin/metadata/filter/IgnoreTsCondition.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/IgnoreTsCondition.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/IgnoreTsCondition.java
new file mode 100644
index 0000000..a074bca
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/IgnoreTsCondition.java
@@ -0,0 +1,64 @@
+package org.apache.kylin.metadata.filter;
+
+import java.util.IdentityHashMap;
+
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.collect.Maps;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 4/13/15.
+ */
+public class IgnoreTsCondition implements TupleFilterSerializer.Decorator {
+
+ private final TblColRef tsColumn;
+ private final TupleFilter root;
+
+ private IdentityHashMap<TupleFilter, Boolean> isInTopLevelANDs;
+
+ public IgnoreTsCondition(TblColRef tsColumn, TupleFilter root) {
+ this.tsColumn = tsColumn;
+ this.root = root;
+ this.isInTopLevelANDs = Maps.newIdentityHashMap();
+ }
+
+ /**
+ * replace filter on timestamp column to null, so that two tuple filter trees can
+ * be compared regardless of the filter condition on timestamp column (In top level where conditions concatenated by ANDs)
+ * @param filter
+ * @return
+ */
+ @Override
+ public TupleFilter onSerialize(TupleFilter filter) {
+
+ if (filter == null)
+ return null;
+
+ //we just need reference equal
+ if (root == filter) {
+ isInTopLevelANDs.put(filter, true);
+ }
+
+ if (isInTopLevelANDs.containsKey(filter)) {
+ classifyChildrenByMarking(filter);
+
+ if (filter instanceof CompareTupleFilter) {
+ if (((CompareTupleFilter) filter).getColumn().equals(tsColumn)) {
+ return null;
+ }
+ }
+ }
+
+ return filter;
+ }
+
+ private void classifyChildrenByMarking(TupleFilter filter) {
+ if (filter instanceof LogicalTupleFilter) {
+ if (filter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
+ for (TupleFilter child : filter.getChildren()) {
+ isInTopLevelANDs.put(child, true);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index 5df3282..fbc26d0 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -69,5 +69,7 @@ public interface IRealization {
public long getDateRangeEnd();
+ public String getModelName();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 8b29a34..f31aa97 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -57,4 +57,5 @@ public class SQLDigest {
"filter on " + this.filterColumns + "," + //
"with aggregates" + this.aggregations + ".";
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
new file mode 100644
index 0000000..ed7acb4
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/StreamSQLDigest.java
@@ -0,0 +1,83 @@
+package org.apache.kylin.metadata.realization;
+
+import java.util.Arrays;
+
+import org.apache.kylin.metadata.filter.IgnoreTsCondition;
+import org.apache.kylin.metadata.filter.TupleFilterSerializer;
+import org.apache.kylin.metadata.model.TblColRef;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 4/13/15.
+ *
+ * A encapsulation of {@link SQLDigest},
+ * This class makes {@link SQLDigest} being able to compare with other {@link SQLDigest}
+ * regardless of the timestamp conditions(In top level where conditions concatenated by ANDs)
+ */
+public class StreamSQLDigest {
+
+ private final SQLDigest sqlDigest;
+
+ private final int hashCode;
+ private final byte[] filterSerialized;
+
+ public StreamSQLDigest(SQLDigest sqlDigest, TblColRef tsCol) {
+ this.sqlDigest = sqlDigest;
+
+ //must use new instance of IgnoreTsCondition
+ IgnoreTsCondition decorator = new IgnoreTsCondition(tsCol, sqlDigest.filter);
+ filterSerialized = TupleFilterSerializer.serialize(sqlDigest.filter, decorator, null);
+
+ int nonFilterHashCode = calculateNonFilterHashCode();
+ this.hashCode = 31 * nonFilterHashCode + (filterSerialized != null ? Arrays.hashCode(filterSerialized) : 0);
+ }
+
+ public byte[] getFilterSerialized() {
+ return filterSerialized;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ StreamSQLDigest other = (StreamSQLDigest) o;
+
+ if (filterSerialized != null ? !Arrays.equals(filterSerialized, other.getFilterSerialized()) : other.getFilterSerialized() != null)
+ return false;
+ if (sqlDigest.aggregations != null ? !sqlDigest.aggregations.equals(other.sqlDigest.aggregations) : other.sqlDigest.aggregations != null)
+ return false;
+ if (sqlDigest.allColumns != null ? !sqlDigest.allColumns.equals(other.sqlDigest.allColumns) : other.sqlDigest.allColumns != null)
+ return false;
+ if (sqlDigest.factTable != null ? !sqlDigest.factTable.equals(other.sqlDigest.factTable) : other.sqlDigest.factTable != null)
+ return false;
+ if (sqlDigest.filterColumns != null ? !sqlDigest.filterColumns.equals(other.sqlDigest.filterColumns) : other.sqlDigest.filterColumns != null)
+ return false;
+ if (sqlDigest.groupbyColumns != null ? !sqlDigest.groupbyColumns.equals(other.sqlDigest.groupbyColumns) : other.sqlDigest.groupbyColumns != null)
+ return false;
+ if (sqlDigest.joinDescs != null ? !sqlDigest.joinDescs.equals(other.sqlDigest.joinDescs) : other.sqlDigest.joinDescs != null)
+ return false;
+ if (sqlDigest.metricColumns != null ? !sqlDigest.metricColumns.equals(other.sqlDigest.metricColumns) : other.sqlDigest.metricColumns != null)
+ return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return this.hashCode;
+ }
+
+ public int calculateNonFilterHashCode() {
+ int result = sqlDigest.factTable != null ? sqlDigest.factTable.hashCode() : 0;
+ result = 31 * result + (sqlDigest.joinDescs != null ? sqlDigest.joinDescs.hashCode() : 0);
+ result = 31 * result + (sqlDigest.allColumns != null ? sqlDigest.allColumns.hashCode() : 0);
+ result = 31 * result + (sqlDigest.groupbyColumns != null ? sqlDigest.groupbyColumns.hashCode() : 0);
+ result = 31 * result + (sqlDigest.filterColumns != null ? sqlDigest.filterColumns.hashCode() : 0);
+ result = 31 * result + (sqlDigest.metricColumns != null ? sqlDigest.metricColumns.hashCode() : 0);
+ result = 31 * result + (sqlDigest.aggregations != null ? sqlDigest.aggregations.hashCode() : 0);
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/storage/pom.xml
----------------------------------------------------------------------
diff --git a/storage/pom.xml b/storage/pom.xml
index 705df50..ef25f91 100644
--- a/storage/pom.xml
+++ b/storage/pom.xml
@@ -51,6 +51,11 @@
<!-- Env & Test -->
+ <dependency>
+ <groupId>net.sf.ehcache</groupId>
+ <artifactId>ehcache</artifactId>
+ <version>2.8.1</version>
+ </dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedTupleIterator.java
new file mode 100644
index 0000000..2b4c549
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedTupleIterator.java
@@ -0,0 +1,71 @@
+package org.apache.kylin.storage.cache;
+
+import com.google.common.base.Preconditions;
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.config.CacheConfiguration;
+import net.sf.ehcache.config.MemoryUnit;
+import net.sf.ehcache.config.PersistenceConfiguration;
+import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.realization.StreamSQLDigest;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.StorageEngineFactory;
+import org.apache.kylin.storage.hbase.CubeSegmentTupleIterator;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 4/13/15.
+ */
+public class CacheFledgedTupleIterator implements IStorageEngine {
+
+ public static final String SUCCESS_QUERY_CACHE = "SuccessQueryCache";
+ public static final String EXCEPTION_QUERY_CACHE = "ExceptionQueryCache";
+
+ static CacheManager cacheManager;
+
+ static {
+ cacheManager = CacheManager.create();
+
+ //Create a Cache specifying its configuration.
+ Cache successCache = new Cache(new CacheConfiguration(SUCCESS_QUERY_CACHE, 0).//
+ memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LFU).//
+ eternal(false).//
+ timeToIdleSeconds(86400).//
+ diskExpiryThreadIntervalSeconds(0).//
+ maxBytesLocalHeap(500, MemoryUnit.MEGABYTES).//
+ persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.LOCALTEMPSWAP)));
+
+ cacheManager.addCache(successCache);
+ }
+
+ private TblColRef partitionColRef;
+
+ public CacheFledgedTupleIterator(IRealization realization) {
+ Preconditions.checkArgument(realization.getType() != RealizationType.CUBE, "Cube realization does not need dynamic cache!");
+ String modelName = realization.getModelName();
+ DataModelDesc dataModelDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getDataModelDesc(modelName);
+ PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
+ Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
+ assert partitionDesc != null;
+ partitionColRef = partitionDesc.getPartitionDateColumnRef();
+ Preconditions.checkArgument(partitionColRef != null, "getPartitionDateColumnRef for " + realization + " is null");
+ }
+
+ @Override
+ public ITupleIterator search(StorageContext context, SQLDigest sqlDigest) {
+ StreamSQLDigest streamSQLDigest = new StreamSQLDigest(sqlDigest, partitionColRef);
+ cacheManager.getCache(SUCCESS_QUERY_CACHE).get(streamSQLDigest).getObjectValue();
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java b/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
new file mode 100644
index 0000000..4d49032
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
@@ -0,0 +1,13 @@
+package org.apache.kylin.storage.cache;
+
+import org.apache.kylin.storage.tuple.Tuple;
+
+import java.util.List;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 4/13/15.
+ */
+public class StreamSQLResult {
+ List<Tuple> rows;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
index a91999f..feb36ca 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
@@ -38,7 +38,7 @@ public class CoprocessorFilter {
byte[] bytes = TupleFilterSerializer.serialize(rootFilter, filterDecorator, DictCodeSystem.INSTANCE);
TupleFilter copy = TupleFilterSerializer.deserialize(bytes, DictCodeSystem.INSTANCE);
- return new CoprocessorFilter(copy, filterDecorator.getUnstrictlyFilteredColumns());
+ return new CoprocessorFilter(copy, filterDecorator.getInevaluableColumns());
}
public static byte[] serialize(CoprocessorFilter o) {
@@ -54,19 +54,19 @@ public class CoprocessorFilter {
// ============================================================================
private final TupleFilter filter;
- private final Set<TblColRef> unstrictlyFilteredColumns;
+ private final Set<TblColRef> inevaluableColumns;
- public CoprocessorFilter(TupleFilter filter, Set<TblColRef> unstrictlyFilteredColumns) {
+ public CoprocessorFilter(TupleFilter filter, Set<TblColRef> inevaluableColumns) {
this.filter = filter;
- this.unstrictlyFilteredColumns = unstrictlyFilteredColumns;
+ this.inevaluableColumns = inevaluableColumns;
}
public TupleFilter getFilter() {
return filter;
}
- public Set<TblColRef> getUnstrictlyFilteredColumns() {
- return unstrictlyFilteredColumns;
+ public Set<TblColRef> getInevaluableColumns() {
+ return inevaluableColumns;
}
public boolean evaluate(IEvaluatableTuple tuple) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
index bbe7c36..563f346 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
@@ -24,7 +24,7 @@ public class DictCodeSystem implements IFilterCodeSystem<String> {
if (value == null)
return true;
- String v = (String) value;
+ String v = value;
for (int i = 0, n = v.length(); i < n; i++) {
if ((byte) v.charAt(i) != Dictionary.NULL)
return false;
@@ -34,12 +34,12 @@ public class DictCodeSystem implements IFilterCodeSystem<String> {
@Override
public int compare(String tupleValue, String constValue) {
- return ((String) tupleValue).compareTo((String) constValue);
+ return tupleValue.compareTo(constValue);
}
@Override
public void serialize(String value, ByteBuffer buffer) {
- BytesUtil.writeUTFString((String) value, buffer);
+ BytesUtil.writeUTFString(value, buffer);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
index 722bd70..139853e 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
@@ -22,17 +22,17 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator {
}
private RowKeyColumnIO columnIO;
- private Set<TblColRef> unstrictlyFilteredColumns;
+ private Set<TblColRef> inevaluableColumns;
private FilterConstantsTreatment filterConstantsTreatment;
public FilterDecorator(IDictionaryAware seg, FilterConstantsTreatment filterConstantsTreatment) {
this.columnIO = new RowKeyColumnIO(seg);
- this.unstrictlyFilteredColumns = Sets.newHashSet();
+ this.inevaluableColumns = Sets.newHashSet();
this.filterConstantsTreatment = filterConstantsTreatment;
}
- public Set<TblColRef> getUnstrictlyFilteredColumns() {
- return unstrictlyFilteredColumns;
+ public Set<TblColRef> getInevaluableColumns() {
+ return inevaluableColumns;
}
@@ -126,11 +126,11 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator {
if (filter == null)
return null;
- // In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE,
- // Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter) will
+ // In case of NOT(inEvaluatableFilter), we should immediately replace it as TRUE,
+ // Otherwise, inEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter) will
// always return FALSE
if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
- TupleFilter.collectColumns(filter, unstrictlyFilteredColumns);
+ TupleFilter.collectColumns(filter, inevaluableColumns);
return ConstantTupleFilter.TRUE;
}
@@ -138,7 +138,7 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator {
return filter;
if (!TupleFilter.isEvaluableRecursively(filter)) {
- TupleFilter.collectColumns(filter, unstrictlyFilteredColumns);
+ TupleFilter.collectColumns(filter, inevaluableColumns);
return ConstantTupleFilter.TRUE;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index 1c26835..43f294c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -120,7 +120,7 @@ public class EndpointTupleIterator implements ITupleIterator {
this.pushedDownRowType = CoprocessorRowType.fromTableRecordInfo(tableRecordInfo, this.columns);
this.pushedDownFilter = CoprocessorFilter.fromFilter(new ClearTextDictionary(this.tableRecordInfo), rootFilter, FilterDecorator.FilterConstantsTreatment.AS_IT_IS);
- for (TblColRef column : this.pushedDownFilter.getUnstrictlyFilteredColumns()) {
+ for (TblColRef column : this.pushedDownFilter.getInevaluableColumns()) {
groupBy.add(column);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index fcdee34..0350489 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -160,6 +160,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
return Math.max(getHistoryRealizationInstance().getDateRangeEnd(), getRealTimeRealizationInstance().getDateRangeEnd()) +1;
}
+ @Override
public String getModelName() {
if (getHistoryRealizationInstance() instanceof CubeInstance) {
return ((CubeInstance) historyRealizationInstance).getDescriptor().getModelName();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java b/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
new file mode 100644
index 0000000..14ebe3e
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
@@ -0,0 +1,46 @@
+package org.apache.kylin.storage.cache;
+
+import com.google.common.collect.Lists;
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.Element;
+import net.sf.ehcache.config.CacheConfiguration;
+import net.sf.ehcache.config.MemoryUnit;
+import net.sf.ehcache.config.PersistenceConfiguration;
+import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.List;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 4/13/15.
+ */
+public class EhcacheTest {
+ @Test
+ public void basicTest() {
+ CacheManager cacheManager = CacheManager.create();
+
+ //Create a Cache specifying its configuration.
+ Cache testCache = new Cache(new CacheConfiguration("test", 0).//
+ memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LFU).//
+ eternal(false).//
+ timeToIdleSeconds(86400).//
+ diskExpiryThreadIntervalSeconds(0).//
+ maxBytesLocalHeap(500, MemoryUnit.MEGABYTES).//
+ persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.LOCALTEMPSWAP)));
+
+ cacheManager.addCache(testCache);
+ testCache.put(new Element("x", Lists.<String> newArrayList()));
+
+ List<String> v = (List<String>) testCache.get("x").getObjectValue();
+ Assert.assertTrue(v.size() == 0);
+ v.add("hi");
+
+ List<String> v2 = (List<String>) testCache.get("x").getObjectValue();
+ Assert.assertTrue(v2.size() == 1);
+
+
+ cacheManager.shutdown();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/storage/src/test/java/org/apache/kylin/storage/filter/IgnoreTsConditionTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/IgnoreTsConditionTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/IgnoreTsConditionTest.java
new file mode 100644
index 0000000..52fc5cd
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/IgnoreTsConditionTest.java
@@ -0,0 +1,133 @@
+package org.apache.kylin.storage.filter;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.index.TableRecordInfo;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.filter.*;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 4/13/15.
+ */
+public class IgnoreTsConditionTest extends LocalFileMetadataTestCase {
+ IIInstance ii;
+ TableRecordInfo tableRecordInfo;
+ CoprocessorFilter filter;
+ TableDesc factTableDesc;
+
+ TblColRef caldt;
+ TblColRef siteId;
+
+ @Before
+ public void setup() throws IOException {
+ this.createTestMetadata();
+ this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
+ this.tableRecordInfo = new TableRecordInfo(ii.getFirstSegment());
+ this.factTableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.TEST_KYLIN_FACT");
+ this.caldt = this.ii.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "CAL_DT");
+ this.siteId = this.ii.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_SITE_ID");
+ }
+
+ @After
+ public void cleanUp() {
+ cleanupTestMetadata();
+ }
+
+ private TupleFilter mockFilter1(int year) {
+ CompareTupleFilter aFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
+ aFilter.addChild(new ColumnTupleFilter(caldt));
+ aFilter.addChild(new ConstantTupleFilter(year + "-01-01"));
+
+ CompareTupleFilter bFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ bFilter.addChild(new ColumnTupleFilter(caldt));
+ bFilter.addChild(new ConstantTupleFilter(year + "-01-04"));
+
+ CompareTupleFilter cFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ cFilter.addChild(new ColumnTupleFilter(caldt));
+ cFilter.addChild(new ConstantTupleFilter(year + "-01-03"));
+
+ CompareTupleFilter dFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ dFilter.addChild(new ColumnTupleFilter(siteId));
+ dFilter.addChild(new ConstantTupleFilter("0"));
+
+ LogicalTupleFilter subRoot = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ subRoot.addChildren(Lists.newArrayList(aFilter, bFilter, cFilter, dFilter));
+
+ CompareTupleFilter outFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ outFilter.addChild(new ColumnTupleFilter(caldt));
+ outFilter.addChild(new ConstantTupleFilter(year + "-01-02"));
+
+ LogicalTupleFilter root = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ root.addChildren(Lists.newArrayList(subRoot, outFilter));
+ return root;
+ }
+
+ private TupleFilter mockFilter2(int year) {
+ CompareTupleFilter aFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
+ aFilter.addChild(new ColumnTupleFilter(caldt));
+ aFilter.addChild(new ConstantTupleFilter(year + "-01-01"));
+
+ CompareTupleFilter bFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ bFilter.addChild(new ColumnTupleFilter(caldt));
+ bFilter.addChild(new ConstantTupleFilter(year + "-01-04"));
+
+ CompareTupleFilter cFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ cFilter.addChild(new ColumnTupleFilter(caldt));
+ cFilter.addChild(new ConstantTupleFilter(year + "-01-03"));
+
+ CompareTupleFilter dFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ dFilter.addChild(new ColumnTupleFilter(siteId));
+ dFilter.addChild(new ConstantTupleFilter("0"));
+
+ LogicalTupleFilter subRoot = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
+ subRoot.addChildren(Lists.newArrayList(aFilter, bFilter, cFilter, dFilter));
+
+ CompareTupleFilter outFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ outFilter.addChild(new ColumnTupleFilter(caldt));
+ outFilter.addChild(new ConstantTupleFilter(year + "-01-02"));
+
+ LogicalTupleFilter root = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ root.addChildren(Lists.newArrayList(subRoot, outFilter));
+ return root;
+ }
+
+ @Test
+ public void positiveTest() {
+
+ TupleFilter a = mockFilter1(2000);
+ TupleFilter b = mockFilter1(2001);
+
+ IgnoreTsCondition decoratorA = new IgnoreTsCondition(caldt, a);
+ byte[] aBytes = TupleFilterSerializer.serialize(a, decoratorA, StringCodeSystem.INSTANCE);
+ IgnoreTsCondition decoratorB = new IgnoreTsCondition(caldt, b);
+ byte[] bBytes = TupleFilterSerializer.serialize(b, decoratorB, StringCodeSystem.INSTANCE);
+ Assert.assertArrayEquals(aBytes, bBytes);
+
+ }
+
+ @Test
+ public void negativeTest()
+ {
+ TupleFilter a = mockFilter2(2000);
+ TupleFilter b = mockFilter2(2001);
+
+ IgnoreTsCondition decoratorA = new IgnoreTsCondition(caldt, a);
+ byte[] aBytes = TupleFilterSerializer.serialize(a, decoratorA, StringCodeSystem.INSTANCE);
+ IgnoreTsCondition decoratorB = new IgnoreTsCondition(caldt, b);
+ byte[] bBytes = TupleFilterSerializer.serialize(b, decoratorB, StringCodeSystem.INSTANCE);
+ Assert.assertFalse(Arrays.equals(aBytes,bBytes));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0e9fbe8/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractorTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractorTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractorTest.java
index d8f1023..32346d5 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractorTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractorTest.java
@@ -56,7 +56,7 @@ public class TsConditionExtractorTest extends LocalFileMetadataTestCase {
this.createTestMetadata();
this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
this.tableRecordInfo = new TableRecordInfo(ii.getFirstSegment());
- factTableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.TEST_KYLIN_FACT");
+ this.factTableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.TEST_KYLIN_FACT");
this.caldt = this.ii.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "CAL_DT");
this.siteId = this.ii.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_SITE_ID");
}
@@ -78,7 +78,7 @@ public class TsConditionExtractorTest extends LocalFileMetadataTestCase {
@Test
- public void testComplextFilter() {
+ public void testComplexFilter() {
CompareTupleFilter aFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
aFilter.addChild(new ColumnTupleFilter(caldt));
aFilter.addChild(new ConstantTupleFilter("2000-01-01"));
@@ -105,7 +105,7 @@ public class TsConditionExtractorTest extends LocalFileMetadataTestCase {
}
@Test
- public void testMoreComplextFilter() {
+ public void testMoreComplexFilter() {
CompareTupleFilter aFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
aFilter.addChild(new ColumnTupleFilter(caldt));
aFilter.addChild(new ConstantTupleFilter("2000-01-01"));
@@ -131,7 +131,7 @@ public class TsConditionExtractorTest extends LocalFileMetadataTestCase {
outFilter.addChild(new ConstantTupleFilter("2000-01-02"));
LogicalTupleFilter root = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
- root.addChildren(Lists.newArrayList(subRoot,outFilter));
+ root.addChildren(Lists.newArrayList(subRoot, outFilter));
Pair<Long, Long> ret = TsConditionExtractor.extractTsCondition(tableRecordInfo, ii.getAllColumns(), root);