You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/14 01:01:18 UTC
[05/50] incubator-kylin git commit: Organize Hybrid codes
Organize Hybrid codes
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/13f32602
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/13f32602
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/13f32602
Branch: refs/heads/streaming
Commit: 13f326029ebd70c91d9ae1e953fbef270fb7f983
Parents: 9609c56
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Fri Mar 6 14:25:14 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Fri Mar 6 14:25:14 2015 +0800
----------------------------------------------------------------------
pom.xml | 3 +-
.../apache/kylin/query/routing/RoutingRule.java | 1 -
.../RoutingRules/RealizationSortRule.java | 21 ++----
.../storage/hybrid/HybridStorageEngine.java | 79 ++++++++------------
4 files changed, 41 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/13f32602/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d59cf0c..e99f77d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -579,7 +579,7 @@
<exclue>**/HiveTableReaderTest.java</exclue>
<exclude>**/TableControllerTest.java</exclude>
<exclude>**/Kafka*Test.java</exclude>
- <exclude>org.apache.kylin.streaming.RequesterTest.java</exclude>
+ <exclude>**/RequesterTest.java</exclude>
</excludes>
<systemProperties>
<property>
@@ -618,6 +618,7 @@
<exclude>**/SampleCubeSetupTest.java</exclude>
<exclude>**/KylinQueryTest.java</exclude>
<exclude>**/Kafka*Test.java</exclude>
+ <exclude>**/RequesterTest.java</exclude>
</excludes>
<systemProperties>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/13f32602/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
index da9a081..ff717a2 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
@@ -42,7 +42,6 @@ public abstract class RoutingRule {
//2. exact match prior to week match
static {
rules.add(new RemoveUncapableRealizationsRule());
- // rules.add(new SimpleQueryMoreColumnsCubeFirstRule());
rules.add(new RealizationSortRule());
rules.add(new AdjustForWeaklyMatchedRealization());//this rule might modify olapcontext content, better put it at last
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/13f32602/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
index d0a96b0..ed8644f 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
@@ -21,6 +21,7 @@ package org.apache.kylin.query.routing.RoutingRules;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.RoutingRule;
+import org.apache.kylin.storage.hybrid.HybridInstance;
import java.util.Collections;
import java.util.Comparator;
@@ -51,22 +52,12 @@ public class RealizationSortRule extends RoutingRule {
return comp;
}
- comp = o1.getAllDimensions().size() - o2.getAllDimensions().size();
- if (comp != 0)
- return comp;
-
- comp = o1.getMeasures().size() - o2.getMeasures().size();
-
- if (comp != 0)
- return comp;
-
- long duration1 = o1.getDateRangeEnd() - o1.getDateRangeStart();
- long duration2 = o2.getDateRangeEnd() - o2.getDateRangeStart();
-
- long diff = duration2 - duration1;
-
+ if (o1 instanceof HybridInstance)
+ return -1;
+ else if (o2 instanceof HybridInstance)
+ return 1;
- return diff == 0 ? 0 : (diff > 0) ? 1 : -1;
+ return 0;
}
});
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/13f32602/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
index 3705c63..7688d68 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
@@ -1,13 +1,10 @@
package org.apache.kylin.storage.hybrid;
-import org.apache.commons.lang.SerializationUtils;
import org.apache.commons.lang.time.FastDateFormat;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.filter.*;
-import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
@@ -18,8 +15,6 @@ import org.apache.kylin.storage.StorageEngineFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collection;
-
/**
* Created by shaoshi on 2/13/15.
*/
@@ -40,54 +35,22 @@ public class HybridStorageEngine implements IStorageEngine {
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest) {
// search the historic realization
-
ITupleIterator iterator1 = searchRealization(hybridInstance.getHistoryRealizationInstance(), context, sqlDigest);
- long boundary = hybridInstance.getHistoryRealizationInstance().getDateRangeEnd();
- FastDateFormat format = FastDateFormat.getInstance("yyyy-MM-dd");
- String boundaryDate = format.format(boundary);
-
- Collection<TblColRef> filterCols = sqlDigest.filterColumns;
-
String modelName = hybridInstance.getModelName();
-
MetadataManager metaMgr = getMetadataManager();
-
DataModelDesc modelDesc = metaMgr.getDataModelDesc(modelName);
+ // if the model isn't partitioned, only query the history
if (modelDesc.getPartitionDesc() == null || modelDesc.getPartitionDesc().getPartitionDateColumnRef() == null)
return iterator1;
- String partitionColFull = modelDesc.getPartitionDesc().getPartitionDateColumn();
-
- String partitionTable = partitionColFull.substring(0, partitionColFull.lastIndexOf("."));
- String partitionCol = partitionColFull.substring(partitionColFull.lastIndexOf(".") + 1);
-
-
- TableDesc factTbl = metaMgr.getTableDesc(partitionTable);
- ColumnDesc columnDesc = factTbl.findColumnByName(partitionCol);
- TblColRef partitionColRef = new TblColRef(columnDesc);
-
-
- // now search the realtime realization, need add the boundary condition
-
-
- CompareTupleFilter compareTupleFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GTE);
- ColumnTupleFilter columnTupleFilter = new ColumnTupleFilter(partitionColRef);
- ConstantTupleFilter constantTupleFilter = new ConstantTupleFilter(boundaryDate);
- compareTupleFilter.addChild(columnTupleFilter);
- compareTupleFilter.addChild(constantTupleFilter);
-
- if (sqlDigest.filter == null) {
- sqlDigest.filter = compareTupleFilter;
- } else {
- LogicalTupleFilter logicalTupleFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ TblColRef partitionColRef = modelDesc.getPartitionDesc().getPartitionDateColumnRef();
- logicalTupleFilter.addChild(sqlDigest.filter);
- logicalTupleFilter.addChild(compareTupleFilter);
+ // add the boundary condition to query real-time
- sqlDigest.filter = logicalTupleFilter;
- }
+ TupleFilter originalFilter = sqlDigest.filter;
+ sqlDigest.filter = createFilterForRealtime(originalFilter, partitionColRef, hybridInstance.getHistoryRealizationInstance().getDateRangeEnd());
boolean addFilterColumn = false, addAllColumn = false;
@@ -101,19 +64,43 @@ public class HybridStorageEngine implements IStorageEngine {
addAllColumn = true;
}
+ // query real-time
ITupleIterator iterator2 = searchRealization(hybridInstance.getRealTimeRealizationInstance(), context, sqlDigest);
// restore the sqlDigest
- sqlDigest.filter = sqlDigest.filter.getChildren().get(0);
+ sqlDigest.filter = originalFilter;
- if(addFilterColumn)
+ if (addFilterColumn)
sqlDigest.filterColumns.remove(partitionColRef);
- if(addAllColumn)
+ if (addAllColumn)
sqlDigest.allColumns.remove(partitionColRef);
+ // combine history and real-time tuple iterator
return new HybridTupleIterator(new ITupleIterator[]{iterator1, iterator2});
-// return new HybridTupleIterator(new ITupleIterator[]{iterator1, ITupleIterator.EMPTY_TUPLE_ITERATOR});
+ }
+
+
+ private TupleFilter createFilterForRealtime(TupleFilter originFilter, TblColRef partitionColRef, long startDate) {
+
+ FastDateFormat format = FastDateFormat.getInstance("yyyy-MM-dd");
+ String boundaryDate = format.format(startDate);
+
+ CompareTupleFilter compareTupleFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GTE);
+ ColumnTupleFilter columnTupleFilter = new ColumnTupleFilter(partitionColRef);
+ ConstantTupleFilter constantTupleFilter = new ConstantTupleFilter(boundaryDate);
+ compareTupleFilter.addChild(columnTupleFilter);
+ compareTupleFilter.addChild(constantTupleFilter);
+
+ if (originFilter == null)
+ return compareTupleFilter;
+
+ LogicalTupleFilter logicalTupleFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+
+ logicalTupleFilter.addChild(originFilter);
+ logicalTupleFilter.addChild(compareTupleFilter);
+
+ return logicalTupleFilter;
}
private ITupleIterator searchRealization(IRealization realization, StorageContext context, SQLDigest sqlDigest) {