You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/08/26 02:39:41 UTC
[1/2] kylin git commit: minor, code refactor
Repository: kylin
Updated Branches:
refs/heads/master 57a259a9d -> 5ee76e8b5
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
index 0f96e3c..7ec24b2 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
@@ -53,19 +53,19 @@ public class CubeTupleConverter {
final CubeSegment cubeSeg;
final Cuboid cuboid;
final TupleInfo tupleInfo;
- final List<IDerivedColumnFiller> derivedColFillers;
+ private final List<IDerivedColumnFiller> derivedColFillers;
- final int[] gtColIdx;
- final int[] tupleIdx;
- final Object[] gtValues;
- final MeasureType<?>[] measureTypes;
+ private final int[] gtColIdx;
+ private final int[] tupleIdx;
+ private final Object[] gtValues;
+ private final MeasureType<?>[] measureTypes;
- final List<IAdvMeasureFiller> advMeasureFillers;
- final List<Integer> advMeasureIndexInGTValues;
+ private final List<IAdvMeasureFiller> advMeasureFillers;
+ private final List<Integer> advMeasureIndexInGTValues;
- final int nSelectedDims;
+ private final int nSelectedDims;
- final int[] dimensionIndexOnTuple;
+ private final int[] dimensionIndexOnTuple;
public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, //
Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo) {
@@ -102,44 +102,44 @@ public class CubeTupleConverter {
////////////
- int iii = 0;
+ int i = 0;
// pre-calculate dimension index mapping to tuple
for (TblColRef dim : selectedDimensions) {
- int i = mapping.getIndexOf(dim);
- gtColIdx[iii] = i;
- tupleIdx[iii] = tupleInfo.hasColumn(dim) ? tupleInfo.getColumnIndex(dim) : -1;
+ int dimIndex = mapping.getIndexOf(dim);
+ gtColIdx[i] = dimIndex;
+ tupleIdx[i] = tupleInfo.hasColumn(dim) ? tupleInfo.getColumnIndex(dim) : -1;
// if (tupleIdx[iii] == -1) {
// throw new IllegalStateException("dim not used in tuple:" + dim);
// }
- iii++;
+ i++;
}
for (FunctionDesc metric : selectedMetrics) {
- int i = mapping.getIndexOf(metric);
- gtColIdx[iii] = i;
+ int metricIndex = mapping.getIndexOf(metric);
+ gtColIdx[i] = metricIndex;
if (metric.needRewrite()) {
String rewriteFieldName = metric.getRewriteFieldName();
- tupleIdx[iii] = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
+ tupleIdx[i] = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
} else {
// a non-rewrite metrics (like sum, or dimension playing as metrics) is like a dimension column
TblColRef col = metric.getParameter().getColRefs().get(0);
- tupleIdx[iii] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
+ tupleIdx[i] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
}
MeasureType<?> measureType = metric.getMeasureType();
if (measureType.needAdvancedTupleFilling()) {
Map<TblColRef, Dictionary<String>> dictionaryMap = buildDictionaryMap(measureType.getColumnsNeedDictionary(metric));
advMeasureFillers.add(measureType.getAdvancedTupleFiller(metric, returnTupleInfo, dictionaryMap));
- advMeasureIndexInGTValues.add(iii);
+ advMeasureIndexInGTValues.add(i);
} else {
- measureTypes[iii] = measureType;
+ measureTypes[i] = measureType;
}
- iii++;
+ i++;
}
// prepare derived columns and filler
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index ae5240b..bacd293 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -127,7 +127,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
if (scanners.isEmpty())
return ITupleIterator.EMPTY_TUPLE_ITERATOR;
- return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context);
+ return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context, cubeDesc.supportsLimitPushDown());
}
protected boolean skipZeroInputSegment(CubeSegment cubeSegment) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index 2cff76c..7059473 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -54,7 +54,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
private int scanCountDelta;
public SequentialCubeTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid, Set<TblColRef> selectedDimensions, //
- Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context) {
+ Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context, boolean supportLimitPushDown) {
this.context = context;
this.scanners = scanners;
@@ -63,8 +63,6 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDimensions, selectedMetrics, returnTupleInfo, context));
}
- boolean supportLimitPushDown = scanners.get(0).getSegment().getCubeDesc().supportsLimitPushDown();
-
this.storagePushDownLimit = context.getStoragePushDownLimit();
if (!supportLimitPushDown || storagePushDownLimit > KylinConfig.getInstanceFromEnv().getStoragePushDownLimitMax()) {
//normal case
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 5db4342..dcacb06 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -51,12 +51,12 @@ public class ITKylinQueryTest extends KylinTestBase {
@BeforeClass
public static void setUp() throws Exception {
+ printInfo("setUp in ITKylinQueryTest");
Map<RealizationType, Integer> priorities = Maps.newHashMap();
priorities.put(RealizationType.HYBRID, 0);
priorities.put(RealizationType.CUBE, 0);
Candidate.setPriorities(priorities);
- printInfo("setUp in ITKylinQueryTest");
joinType = "left";
setupAll();
@@ -64,8 +64,8 @@ public class ITKylinQueryTest extends KylinTestBase {
@AfterClass
public static void tearDown() throws Exception {
- Candidate.restorePriorities();
printInfo("tearDown in ITKylinQueryTest");
+ Candidate.restorePriorities();
clean();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0a82bf8..ea29958 100644
--- a/pom.xml
+++ b/pom.xml
@@ -366,6 +366,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <version>${hadoop2.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
<version>${hadoop2.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index bf93300..96e75f3 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -25,6 +25,7 @@ import org.apache.calcite.DataContext;
import org.apache.calcite.jdbc.CalciteConnection;
import org.apache.calcite.linq4j.Enumerator;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -162,8 +163,14 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
// Hack no-group-by query for better results
private void hackNoGroupByAggregation(SQLDigest sqlDigest) {
- if (!sqlDigest.groupbyColumns.isEmpty() || !sqlDigest.metricColumns.isEmpty())
+ if (!(olapContext.realization instanceof CubeInstance)) {
+ //the hack only makes sense for cubes
return;
+ }
+
+ if (!sqlDigest.isRawQuery()) {
+ return;
+ }
// If no group by and metric found, then it's simple query like select ... from ... where ...,
// But we have no raw data stored, in order to return better results, we hack to output sum of metric column
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
index ab7884a..9ea8961 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
@@ -35,6 +35,7 @@ public class Candidate implements Comparable<Candidate> {
static {
DEFAULT_PRIORITIES.put(RealizationType.HYBRID, 0);
DEFAULT_PRIORITIES.put(RealizationType.CUBE, 1);
+ DEFAULT_PRIORITIES.put(RealizationType.INVERTED_INDEX, 1);
}
/** for test only */
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 200c040..07a3cc3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -40,7 +40,7 @@ import org.apache.kylin.common.util.CompressionUtils;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.LoggableCachedThreadPool;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.ISegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTScanRange;
@@ -67,8 +67,8 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
private static ExecutorService executorService = new LoggableCachedThreadPool();
- public CubeHBaseEndpointRPC(CubeSegment cubeSeg, Cuboid cuboid, GTInfo fullGTInfo) {
- super(cubeSeg, cuboid, fullGTInfo);
+ public CubeHBaseEndpointRPC(ISegment segment, Cuboid cuboid, GTInfo fullGTInfo) {
+ super(segment, cuboid, fullGTInfo);
}
private byte[] getByteArrayForShort(short v) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 687ee83..c318cba 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -31,6 +31,7 @@ import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.ISegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.FuzzyKeyEncoder;
import org.apache.kylin.cube.kv.FuzzyMaskEncoder;
@@ -60,8 +61,10 @@ public abstract class CubeHBaseRPC implements IGTStorage {
final private RowKeyEncoder fuzzyKeyEncoder;
final private RowKeyEncoder fuzzyMaskEncoder;
- public CubeHBaseRPC(CubeSegment cubeSeg, Cuboid cuboid, GTInfo fullGTInfo) {
- this.cubeSeg = cubeSeg;
+ public CubeHBaseRPC(ISegment segment, Cuboid cuboid, GTInfo fullGTInfo) {
+ Preconditions.checkArgument(segment instanceof CubeSegment, "segment must be CubeSegment");
+
+ this.cubeSeg = (CubeSegment) segment;
this.cuboid = cuboid;
this.fullGTInfo = fullGTInfo;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 238939c..a359d19 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.ShardingHash;
-import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.ISegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.dimension.DimensionEncoding;
@@ -89,8 +89,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
}
}
- public CubeHBaseScanRPC(CubeSegment cubeSeg, Cuboid cuboid, final GTInfo fullGTInfo) {
- super(cubeSeg, cuboid, fullGTInfo);
+ public CubeHBaseScanRPC(ISegment segment, Cuboid cuboid, final GTInfo fullGTInfo) {
+ super(segment, cuboid, fullGTInfo);
MassInTupleFilter.VALUE_PROVIDER_FACTORY = new MassInValueProviderFactoryImpl(new MassInValueProviderFactoryImpl.DimEncAware() {
@Override
public DimensionEncoding getDimEnc(TblColRef col) {
[2/2] kylin git commit: minor, code refactor
Posted by ma...@apache.org.
minor, code refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5ee76e8b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5ee76e8b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5ee76e8b
Branch: refs/heads/master
Commit: 5ee76e8b5840e7f8c6da9a56e6f1f496f0821589
Parents: 57a259a
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Aug 25 14:17:19 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Aug 26 10:39:19 2016 +0800
----------------------------------------------------------------------
.../kylin/cube/CubeCapabilityChecker.java | 50 +-
.../org/apache/kylin/cube/CubeInstance.java | 2 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 10 +-
.../java/org/apache/kylin/cube/ISegment.java | 39 ++
.../java/org/apache/kylin/cube/JoinChecker.java | 68 +++
.../gridtable/AsymmetricRecordComparator.java | 55 ++
.../kylin/cube/gridtable/ComparatorEx.java | 64 +++
.../kylin/cube/gridtable/CubeGridTable.java | 21 -
.../cube/gridtable/CubeScanRangePlanner.java | 542 +------------------
.../gridtable/CuboidToGridTableMapping.java | 60 +-
.../kylin/cube/gridtable/RecordComparator.java | 46 ++
.../kylin/cube/gridtable/RecordComparators.java | 77 +++
.../cube/gridtable/ScanRangePlannerBase.java | 295 ++++++++++
.../cube/gridtable/SegmentGTStartAndEnd.java | 84 +++
.../java/org/apache/kylin/gridtable/GTInfo.java | 16 +-
.../apache/kylin/gridtable/GTScanRequest.java | 16 +-
.../kylin/gridtable/GTScanRequestBuilder.java | 13 +
.../apache/kylin/gridtable/ScannerWorker.java | 8 +-
.../kylin/gridtable/DictGridTableTest.java | 2 +-
.../kylin/metadata/project/ProjectL2Cache.java | 8 +
.../kylin/metadata/realization/SQLDigest.java | 6 +-
.../storage/gtrecord/CubeTupleConverter.java | 42 +-
.../gtrecord/GTCubeStorageQueryBase.java | 2 +-
.../gtrecord/SequentialCubeTupleIterator.java | 4 +-
.../apache/kylin/query/ITKylinQueryTest.java | 4 +-
pom.xml | 6 +
.../kylin/query/enumerator/OLAPEnumerator.java | 9 +-
.../apache/kylin/query/routing/Candidate.java | 1 +
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 6 +-
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 7 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 6 +-
31 files changed, 919 insertions(+), 650 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 1f16b1b..e0d8dd3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -18,7 +18,6 @@
package org.apache.kylin.cube;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
@@ -26,18 +25,16 @@ import java.util.List;
import java.util.Set;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.basic.BasicMeasureType;
import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IStorageAware;
-import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,7 +50,7 @@ public class CubeCapabilityChecker {
result.capable = false;
// match joins
- boolean isJoinMatch = isJoinMatch(digest.joinDescs, cube);
+ boolean isJoinMatch = JoinChecker.isJoinMatch(digest.joinDescs, cube);
if (!isJoinMatch) {
logger.info("Exclude cube " + cube.getName() + " because unmatched joins");
return result;
@@ -89,7 +86,15 @@ public class CubeCapabilityChecker {
if (cube.getStorageType() == IStorageAware.ID_HBASE && MassInTupleFilter.containsMassInTupleFilter(digest.filter)) {
logger.info("Exclude cube " + cube.getName() + " because only v2 storage + v2 query engine supports massin");
return result;
+ }
+ if (digest.isRawQuery() && cube.getFactTable().equals(digest.factTable)) {
+ result.influences.add(new CapabilityInfluence() {
+ @Override
+ public double suggestCostMultiplier() {
+ return 100;
+ }
+ });
}
// cost will be minded by caller
@@ -121,41 +126,6 @@ public class CubeCapabilityChecker {
return result;
}
- private static boolean isJoinMatch(Collection<JoinDesc> joins, CubeInstance cube) {
- CubeDesc cubeDesc = cube.getDescriptor();
-
- List<JoinDesc> cubeJoins = new ArrayList<JoinDesc>(cubeDesc.getDimensions().size());
- for (DimensionDesc d : cubeDesc.getDimensions()) {
- if (d.getJoin() != null) {
- cubeJoins.add(d.getJoin());
- }
- }
- for (JoinDesc j : joins) {
- // optiq engine can't decide which one is fk or pk
- String pTable = j.getPrimaryKeyColumns()[0].getTable();
- String factTable = cubeDesc.getFactTable();
- if (factTable.equals(pTable)) {
- j.swapPKFK();
- }
-
- // check primary key, all PK column should refer to same tale, the Fact Table of cube.
- // Using first column's table name to check.
- String fTable = j.getForeignKeyColumns()[0].getTable();
- if (!factTable.equals(fTable)) {
- logger.info("Fact Table" + factTable + " not matched in join: " + j + " on cube " + cube.getName());
- return false;
- }
-
- // The hashcode() function of JoinDesc has been overwritten,
- // which takes into consideration: pk,fk,jointype
- if (!cubeJoins.contains(j)) {
- logger.info("Query joins don't macth on cube " + cube.getName());
- return false;
- }
- }
- return true;
- }
-
private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult result) {
CubeDesc cubeDesc = cube.getDescriptor();
Collection<FunctionDesc> cubeFuncs = cubeDesc.listAllFunctions();
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 90da59d..151e142 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -352,7 +352,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return result;
}
- private int getCost(SQLDigest digest) {
+ public int getCost(SQLDigest digest) {
int calculatedCost = cost;
//the number of dimensions is not as accurate as number of row key cols
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index aaa88f1..79397c3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -35,21 +35,22 @@ import org.apache.kylin.common.util.ShardingHash;
import org.apache.kylin.cube.kv.CubeDimEncMap;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IBuildable;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonBackReference;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeSegment implements Comparable<CubeSegment>, IBuildable {
+public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegment {
@JsonBackReference
private CubeInstance cubeInstance;
@@ -175,6 +176,11 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable {
return status;
}
+ @Override
+ public DataModelDesc getModel() {
+ return this.getCubeDesc().getModel();
+ }
+
public void setStatus(SegmentStatusEnum status) {
this.status = status;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java b/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java
new file mode 100644
index 0000000..2e1f214
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube;
+
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+
+public interface ISegment {
+
+ public String getName();
+
+ public long getDateRangeStart();
+
+ public long getDateRangeEnd();
+
+ public long getSourceOffsetStart();
+
+ public long getSourceOffsetEnd();
+
+ public DataModelDesc getModel();
+
+ public SegmentStatusEnum getStatus();
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java
new file mode 100644
index 0000000..edd5be9
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.LookupDesc;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+public class JoinChecker {
+
+ private static final Logger logger = LoggerFactory.getLogger(CubeCapabilityChecker.class);
+
+ public static boolean isJoinMatch(Collection<JoinDesc> joins, IRealization realization) {
+
+ List<JoinDesc> realizationsJoins = Lists.newArrayList();
+ for (LookupDesc lookupDesc : realization.getDataModelDesc().getLookups()) {
+ realizationsJoins.add(lookupDesc.getJoin());
+ }
+
+ for (JoinDesc j : joins) {
+ // optiq engine can't decide which one is fk or pk
+ String pTable = j.getPrimaryKeyColumns()[0].getTable();
+ String factTable = realization.getFactTable();
+ if (factTable.equals(pTable)) {
+ j.swapPKFK();
+ }
+
+ // check primary key, all PK column should refer to same tale, the Fact Table of cube.
+ // Using first column's table name to check.
+ String fTable = j.getForeignKeyColumns()[0].getTable();
+ if (!factTable.equals(fTable)) {
+ logger.info("Fact Table" + factTable + " not matched in join: " + j + " on cube " + realization.getName());
+ return false;
+ }
+
+ // The hashcode() function of JoinDesc has been overwritten,
+ // which takes into consideration: pk,fk,jointype
+ if (!realizationsJoins.contains(j)) {
+ logger.info("Query joins don't macth on cube " + realization.getName());
+ return false;
+ }
+ }
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/AsymmetricRecordComparator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/AsymmetricRecordComparator.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/AsymmetricRecordComparator.java
new file mode 100644
index 0000000..74dc855
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/AsymmetricRecordComparator.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.gridtable;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.gridtable.GTRecord;
+
+import java.util.Collection;
+
+/**
+ * asymmetric means compare(a,b) > 0 does not cause compare(b,a) < 0
+ * so min max functions will not be supported
+ */
+public class AsymmetricRecordComparator extends RecordComparator {
+
+ AsymmetricRecordComparator(ComparatorEx<ByteArray> byteComparator) {
+ super(byteComparator);
+ }
+
+ public GTRecord min(Collection<GTRecord> v) {
+ throw new UnsupportedOperationException();
+ }
+
+ public GTRecord max(Collection<GTRecord> v) {
+ throw new UnsupportedOperationException();
+ }
+
+ public GTRecord min(GTRecord a, GTRecord b) {
+ throw new UnsupportedOperationException();
+ }
+
+ public GTRecord max(GTRecord a, GTRecord b) {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean between(GTRecord v, GTRecord start, GTRecord end) {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ComparatorEx.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ComparatorEx.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ComparatorEx.java
new file mode 100644
index 0000000..3f1e8af
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ComparatorEx.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.gridtable;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+
+public abstract class ComparatorEx<T> implements Comparator<T> {
+
+ public T min(Collection<T> v) {
+ if (v.size() <= 0) {
+ return null;
+ }
+
+ Iterator<T> iterator = v.iterator();
+ T min = iterator.next();
+ while (iterator.hasNext()) {
+ min = min(min, iterator.next());
+ }
+ return min;
+ }
+
+ public T max(Collection<T> v) {
+ if (v.size() <= 0) {
+ return null;
+ }
+
+ Iterator<T> iterator = v.iterator();
+ T max = iterator.next();
+ while (iterator.hasNext()) {
+ max = max(max, iterator.next());
+ }
+ return max;
+ }
+
+ public T min(T a, T b) {
+ return compare(a, b) <= 0 ? a : b;
+ }
+
+ public T max(T a, T b) {
+ return compare(a, b) >= 0 ? a : b;
+ }
+
+ public boolean between(T v, T start, T end) {
+ return compare(start, v) <= 0 && compare(v, end) <= 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
index f2f3a25..563cf43 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
@@ -18,11 +18,9 @@
package org.apache.kylin.cube.gridtable;
-import java.util.List;
import java.util.Map;
import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.CubeDimEncMap;
@@ -31,27 +29,8 @@ import org.apache.kylin.dimension.IDimensionEncodingMap;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.metadata.model.TblColRef;
-import com.google.common.collect.Maps;
-
public class CubeGridTable {
- public static Map<TblColRef, Dictionary<String>> getDimensionToDictionaryMap(CubeSegment cubeSeg, long cuboidId) {
- CubeDesc cubeDesc = cubeSeg.getCubeDesc();
- CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig());
-
- // build a dictionary map
- Map<TblColRef, Dictionary<String>> dictionaryMap = Maps.newHashMap();
- List<TblColRef> dimCols = Cuboid.findById(cubeDesc, cuboidId).getColumns();
- for (TblColRef col : dimCols) {
- Dictionary<String> dictionary = cubeMgr.getDictionary(cubeSeg, col);
- if (dictionary != null) {
- dictionaryMap.put(col, dictionary);
- }
- }
-
- return dictionaryMap;
- }
-
public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) {
Cuboid cuboid = Cuboid.findById(cubeSeg.getCubeDesc(), cuboidId);
return newGTInfo(cuboid, new CubeDimEncMap(cubeSeg));
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java
index 58771da..a937045 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java
@@ -18,25 +18,17 @@
package org.apache.kylin.cube.gridtable;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.BitSet;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.common.FuzzyValueCombination;
@@ -49,12 +41,7 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTScanRequestBuilder;
import org.apache.kylin.gridtable.GTUtil;
import org.apache.kylin.gridtable.IGTComparator;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -64,7 +51,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-public class CubeScanRangePlanner {
+public class CubeScanRangePlanner extends ScanRangePlannerBase {
private static final Logger logger = LoggerFactory.getLogger(CubeScanRangePlanner.class);
@@ -75,24 +62,6 @@ public class CubeScanRangePlanner {
protected CubeSegment cubeSegment;
protected CubeDesc cubeDesc;
protected Cuboid cuboid;
- protected TupleFilter filter;
- protected Set<TblColRef> dimensions;
- protected Set<TblColRef> groupbyDims;
- protected Set<TblColRef> filterDims;
- protected Collection<FunctionDesc> metrics;
-
- //GT
- protected TupleFilter gtFilter;
- protected GTInfo gtInfo;
- protected Pair<ByteArray, ByteArray> gtStartAndEnd;
- protected TblColRef gtPartitionCol;
- protected ImmutableBitSet gtDimensions;
- protected ImmutableBitSet gtAggrGroups;
- protected ImmutableBitSet gtAggrMetrics;
- protected String[] gtAggrFuncs;
- final protected RecordComparator rangeStartComparator;
- final protected RecordComparator rangeEndComparator;
- final protected RecordComparator rangeStartEndComparator;
public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, TupleFilter filter, Set<TblColRef> dimensions, Set<TblColRef> groupbyDims, //
Collection<FunctionDesc> metrics) {
@@ -103,36 +72,35 @@ public class CubeScanRangePlanner {
this.cubeSegment = cubeSegment;
this.cubeDesc = cubeSegment.getCubeDesc();
this.cuboid = cuboid;
- this.dimensions = dimensions;
- this.groupbyDims = groupbyDims;
- this.filter = filter;
- this.metrics = metrics;
- this.filterDims = Sets.newHashSet();
- TupleFilter.collectColumns(filter, this.filterDims);
+
+ Set<TblColRef> filterDims = Sets.newHashSet();
+ TupleFilter.collectColumns(filter, filterDims);
this.gtInfo = CubeGridTable.newGTInfo(cubeSegment, cuboid.getId());
CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
IGTComparator comp = gtInfo.getCodeSystem().getComparator();
//start key GTRecord compare to start key GTRecord
- this.rangeStartComparator = getRangeStartComparator(comp);
+ this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp);
//stop key GTRecord compare to stop key GTRecord
- this.rangeEndComparator = getRangeEndComparator(comp);
+ this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp);
//start key GTRecord compare to stop key GTRecord
- this.rangeStartEndComparator = getRangeStartEndComparator(comp);
+ this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp);
//replace the constant values in filter to dictionary codes
- this.gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, gtInfo, mapping.getCuboidDimensionsInGTOrder(), this.groupbyDims);
+ this.gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, gtInfo, mapping.getCuboidDimensionsInGTOrder(), groupbyDims);
- this.gtDimensions = makeGridTableColumns(mapping, dimensions);
- this.gtAggrGroups = makeGridTableColumns(mapping, replaceDerivedColumns(groupbyDims, cubeSegment.getCubeDesc()));
- this.gtAggrMetrics = makeGridTableColumns(mapping, metrics);
- this.gtAggrFuncs = makeAggrFuncs(mapping, metrics);
+ this.gtDimensions = mapping.makeGridTableColumns(dimensions);
+ this.gtAggrGroups = mapping.makeGridTableColumns(replaceDerivedColumns(groupbyDims, cubeSegment.getCubeDesc()));
+ this.gtAggrMetrics = mapping.makeGridTableColumns(metrics);
+ this.gtAggrFuncs = mapping.makeAggrFuncs(metrics);
- if (cubeSegment.getCubeDesc().getModel().getPartitionDesc().isPartitioned()) {
- int index = mapping.getIndexOf(cubeSegment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef());
+ if (cubeSegment.getModel().getPartitionDesc().isPartitioned()) {
+ int index = mapping.getIndexOf(cubeSegment.getModel().getPartitionDesc().getPartitionDateColumnRef());
if (index >= 0) {
- this.gtStartAndEnd = getSegmentStartAndEnd(index);
+ SegmentGTStartAndEnd segmentGTStartAndEnd = new SegmentGTStartAndEnd(cubeSegment, gtInfo);
+ this.gtStartAndEnd = segmentGTStartAndEnd.getSegmentStartAndEnd(index);
+ this.isPartitionColUsingDatetimeEncoding = segmentGTStartAndEnd.isUsingDatetimeEncoding(index);
this.gtPartitionCol = gtInfo.colRef(index);
}
}
@@ -155,11 +123,11 @@ public class CubeScanRangePlanner {
IGTComparator comp = gtInfo.getCodeSystem().getComparator();
//start key GTRecord compare to start key GTRecord
- this.rangeStartComparator = getRangeStartComparator(comp);
+ this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp);
//stop key GTRecord compare to stop key GTRecord
- this.rangeEndComparator = getRangeEndComparator(comp);
+ this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp);
//start key GTRecord compare to stop key GTRecord
- this.rangeStartEndComparator = getRangeStartEndComparator(comp);
+ this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp);
this.gtFilter = gtFilter;
this.gtStartAndEnd = gtStartAndEnd;
@@ -199,46 +167,6 @@ public class CubeScanRangePlanner {
return mergedRanges;
}
- private Pair<ByteArray, ByteArray> getSegmentStartAndEnd(int index) {
- ByteArray start;
- if (cubeSegment.getDateRangeStart() != Long.MIN_VALUE) {
- start = encodeTime(cubeSegment.getDateRangeStart(), index, 1);
- } else {
- start = new ByteArray();
- }
-
- ByteArray end;
- if (cubeSegment.getDateRangeEnd() != Long.MAX_VALUE) {
- end = encodeTime(cubeSegment.getDateRangeEnd(), index, -1);
- } else {
- end = new ByteArray();
- }
- return Pair.newPair(start, end);
-
- }
-
- private ByteArray encodeTime(long ts, int index, int roundingFlag) {
- String value;
- DataType partitionColType = gtInfo.getColumnType(index);
- if (partitionColType.isDate()) {
- value = DateFormat.formatToDateStr(ts);
- } else if (partitionColType.isDatetime() || partitionColType.isTimestamp()) {
- value = DateFormat.formatToTimeWithoutMilliStr(ts);
- } else if (partitionColType.isStringFamily()) {
- String partitionDateFormat = cubeSegment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateFormat();
- if (StringUtils.isEmpty(partitionDateFormat))
- partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN;
- value = DateFormat.formatToDateStr(ts, partitionDateFormat);
- } else {
- throw new RuntimeException("Type " + partitionColType + " is not valid partition column type");
- }
-
- ByteBuffer buffer = ByteBuffer.allocate(gtInfo.getMaxColumnLength());
- gtInfo.getCodeSystem().encodeColumnValue(index, value, roundingFlag, buffer);
-
- return ByteArray.copyOf(buffer.array(), 0, buffer.position());
- }
-
private Set<TblColRef> replaceDerivedColumns(Set<TblColRef> input, CubeDesc cubeDesc) {
Set<TblColRef> ret = Sets.newHashSet();
for (TblColRef col : input) {
@@ -253,57 +181,6 @@ public class CubeScanRangePlanner {
return ret;
}
- private ImmutableBitSet makeGridTableColumns(CuboidToGridTableMapping mapping, Set<TblColRef> dimensions) {
- BitSet result = new BitSet();
- for (TblColRef dim : dimensions) {
- int idx = mapping.getIndexOf(dim);
- if (idx >= 0)
- result.set(idx);
- }
- return new ImmutableBitSet(result);
- }
-
- private ImmutableBitSet makeGridTableColumns(CuboidToGridTableMapping mapping, Collection<FunctionDesc> metrics) {
- BitSet result = new BitSet();
- for (FunctionDesc metric : metrics) {
- int idx = mapping.getIndexOf(metric);
- if (idx < 0)
- throw new IllegalStateException(metric + " not found in " + mapping);
- result.set(idx);
- }
- return new ImmutableBitSet(result);
- }
-
- private String[] makeAggrFuncs(final CuboidToGridTableMapping mapping, Collection<FunctionDesc> metrics) {
-
- //metrics are represented in ImmutableBitSet, which loses order information
- //sort the aggrFuns to align with metrics natural order
- List<FunctionDesc> metricList = Lists.newArrayList(metrics);
- Collections.sort(metricList, new Comparator<FunctionDesc>() {
- @Override
- public int compare(FunctionDesc o1, FunctionDesc o2) {
- int a = mapping.getIndexOf(o1);
- int b = mapping.getIndexOf(o2);
- return a - b;
- }
- });
-
- String[] result = new String[metricList.size()];
- int i = 0;
- for (FunctionDesc metric : metricList) {
- result[i++] = metric.getExpression();
- }
- return result;
- }
-
- private String makeReadable(ByteArray byteArray) {
- if (byteArray == null) {
- return null;
- } else {
- return byteArray.toReadableText();
- }
- }
-
protected GTScanRange newScanRange(Collection<ColumnRange> andDimRanges) {
GTRecord pkStart = new GTRecord(gtInfo);
GTRecord pkEnd = new GTRecord(gtInfo);
@@ -313,14 +190,14 @@ public class CubeScanRangePlanner {
for (ColumnRange range : andDimRanges) {
if (gtPartitionCol != null && range.column.equals(gtPartitionCol)) {
- if (rangeStartEndComparator.comparator.compare(gtStartAndEnd.getFirst(), range.end) <= 0 //
- && (rangeStartEndComparator.comparator.compare(range.begin, gtStartAndEnd.getSecond()) < 0 //
- || rangeStartEndComparator.comparator.compare(range.begin, gtStartAndEnd.getSecond()) == 0 //
- && (range.op == FilterOperatorEnum.EQ || range.op == FilterOperatorEnum.LTE || range.op == FilterOperatorEnum.GTE || range.op == FilterOperatorEnum.IN))) {
- //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded, so use <= when has equals in condition.
+ int beginCompare = rangeStartEndComparator.comparator.compare(range.begin, gtStartAndEnd.getSecond());
+ int endCompare = rangeStartEndComparator.comparator.compare(gtStartAndEnd.getFirst(), range.end);
+
+ if ((isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare < 0) || (!isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare <= 0)) {
+ //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded when using dict encoding, so use <= when has equals in condition.
} else {
logger.debug("Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}", //
- new Object[] { gtPartitionCol, makeReadable(gtStartAndEnd.getFirst()), makeReadable(gtStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end) });
+ gtPartitionCol, makeReadable(gtStartAndEnd.getFirst()), makeReadable(gtStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end));
return null;
}
}
@@ -337,7 +214,9 @@ public class CubeScanRangePlanner {
}
}
- fuzzyKeys = buildFuzzyKeys(fuzzyValues);
+ fuzzyKeys =
+
+ buildFuzzyKeys(fuzzyValues);
return new GTScanRange(pkStart, pkEnd, fuzzyKeys);
}
@@ -371,104 +250,6 @@ public class CubeScanRangePlanner {
return result;
}
- protected TupleFilter flattenToOrAndFilter(TupleFilter filter) {
- if (filter == null)
- return null;
-
- TupleFilter flatFilter = filter.flatFilter();
-
- // normalize to OR-AND filter
- if (flatFilter.getOperator() == FilterOperatorEnum.AND) {
- LogicalTupleFilter f = new LogicalTupleFilter(FilterOperatorEnum.OR);
- f.addChild(flatFilter);
- flatFilter = f;
- }
-
- if (flatFilter.getOperator() != FilterOperatorEnum.OR)
- throw new IllegalStateException();
-
- return flatFilter;
- }
-
- protected List<Collection<ColumnRange>> translateToOrAndDimRanges(TupleFilter flatFilter) {
- List<Collection<ColumnRange>> result = Lists.newArrayList();
-
- if (flatFilter == null) {
- result.add(Collections.<ColumnRange> emptyList());
- return result;
- }
-
- for (TupleFilter andFilter : flatFilter.getChildren()) {
- if (andFilter.getOperator() != FilterOperatorEnum.AND)
- throw new IllegalStateException("Filter should be AND instead of " + andFilter);
-
- Collection<ColumnRange> andRanges = translateToAndDimRanges(andFilter.getChildren());
- if (andRanges != null) {
- result.add(andRanges);
- }
- }
-
- return preEvaluateConstantConditions(result);
- }
-
- private Collection<ColumnRange> translateToAndDimRanges(List<? extends TupleFilter> andFilters) {
- Map<TblColRef, ColumnRange> rangeMap = new HashMap<TblColRef, ColumnRange>();
- for (TupleFilter filter : andFilters) {
- if ((filter instanceof CompareTupleFilter) == false) {
- if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) {
- return null;
- } else {
- continue;
- }
- }
-
- CompareTupleFilter comp = (CompareTupleFilter) filter;
- if (comp.getColumn() == null) {
- continue;
- }
-
- @SuppressWarnings("unchecked")
- ColumnRange newRange = new ColumnRange(comp.getColumn(), (Set<ByteArray>) comp.getValues(), comp.getOperator());
- ColumnRange existing = rangeMap.get(newRange.column);
- if (existing == null) {
- rangeMap.put(newRange.column, newRange);
- } else {
- existing.andMerge(newRange);
- }
- }
- return rangeMap.values();
- }
-
- private List<Collection<ColumnRange>> preEvaluateConstantConditions(List<Collection<ColumnRange>> orAndRanges) {
- boolean globalAlwaysTrue = false;
- Iterator<Collection<ColumnRange>> iterator = orAndRanges.iterator();
- while (iterator.hasNext()) {
- Collection<ColumnRange> andRanges = iterator.next();
- Iterator<ColumnRange> iterator2 = andRanges.iterator();
- boolean hasAlwaysFalse = false;
- while (iterator2.hasNext()) {
- ColumnRange range = iterator2.next();
- if (range.satisfyAll())
- iterator2.remove();
- else if (range.satisfyNone())
- hasAlwaysFalse = true;
- }
- if (hasAlwaysFalse) {
- iterator.remove();
- } else if (andRanges.isEmpty()) {
- globalAlwaysTrue = true;
- break;
- }
- }
- // return empty OR list means global false
- // return an empty AND collection inside OR list means global true
- if (globalAlwaysTrue) {
- orAndRanges.clear();
- orAndRanges.add(Collections.<ColumnRange> emptyList());
- }
- return orAndRanges;
- }
-
protected List<GTScanRange> mergeOverlapRanges(List<GTScanRange> ranges) {
if (ranges.size() <= 1) {
return ranges;
@@ -556,269 +337,4 @@ public class CubeScanRangePlanner {
this.maxScanRanges = maxScanRanges;
}
- protected class ColumnRange {
- private TblColRef column;
- private ByteArray begin = ByteArray.EMPTY;
- private ByteArray end = ByteArray.EMPTY;
- private Set<ByteArray> valueSet;
- private FilterOperatorEnum op;
-
- public ColumnRange(TblColRef column, Set<ByteArray> values, FilterOperatorEnum op) {
- this.column = column;
- this.op = op;
-
- switch (op) {
- case EQ:
- case IN:
- valueSet = new HashSet<ByteArray>(values);
- refreshBeginEndFromEquals();
- break;
- case LT:
- case LTE:
- end = rangeEndComparator.comparator.max(values);
- break;
- case GT:
- case GTE:
- begin = rangeStartComparator.comparator.min(values);
- break;
- case NEQ:
- case NOTIN:
- case ISNULL:
- case ISNOTNULL:
- // let Optiq filter it!
- break;
- default:
- throw new UnsupportedOperationException(op.name());
- }
- }
-
- void copy(TblColRef column, ByteArray beginValue, ByteArray endValue, Set<ByteArray> equalValues) {
- this.column = column;
- this.begin = beginValue;
- this.end = endValue;
- this.valueSet = equalValues;
- }
-
- private void refreshBeginEndFromEquals() {
- if (valueSet.isEmpty()) {
- begin = ByteArray.EMPTY;
- end = ByteArray.EMPTY;
- } else {
- begin = rangeStartComparator.comparator.min(valueSet);
- end = rangeEndComparator.comparator.max(valueSet);
- }
- }
-
- public boolean satisfyAll() {
- return begin.array() == null && end.array() == null; // the NEQ case
- }
-
- public boolean satisfyNone() {
- if (valueSet != null) {
- return valueSet.isEmpty();
- } else if (begin.array() != null && end.array() != null) {
- return gtInfo.getCodeSystem().getComparator().compare(begin, end) > 0;
- } else {
- return false;
- }
- }
-
- public void andMerge(ColumnRange another) {
- assert this.column.equals(another.column);
-
- if (another.satisfyAll()) {
- return;
- }
-
- if (this.satisfyAll()) {
- copy(another.column, another.begin, another.end, another.valueSet);
- return;
- }
-
- if (this.valueSet != null && another.valueSet != null) {
- this.valueSet.retainAll(another.valueSet);
- refreshBeginEndFromEquals();
- return;
- }
-
- if (this.valueSet != null) {
- this.valueSet = filter(this.valueSet, another.begin, another.end);
- refreshBeginEndFromEquals();
- return;
- }
-
- if (another.valueSet != null) {
- this.valueSet = filter(another.valueSet, this.begin, this.end);
- refreshBeginEndFromEquals();
- return;
- }
-
- this.begin = rangeStartComparator.comparator.max(this.begin, another.begin);
- this.end = rangeEndComparator.comparator.min(this.end, another.end);
- }
-
- private Set<ByteArray> filter(Set<ByteArray> equalValues, ByteArray beginValue, ByteArray endValue) {
- Set<ByteArray> result = Sets.newHashSetWithExpectedSize(equalValues.size());
- for (ByteArray v : equalValues) {
- if (rangeStartEndComparator.comparator.compare(beginValue, v) <= 0 && rangeStartEndComparator.comparator.compare(v, endValue) <= 0) {
- result.add(v);
- }
- }
- return equalValues;
- }
-
- public String toString() {
- if (valueSet == null) {
- return column.getName() + " between " + begin + " and " + end;
- } else {
- return column.getName() + " in " + valueSet;
- }
- }
- }
-
- public static abstract class ComparatorEx<T> implements Comparator<T> {
-
- public T min(Collection<T> v) {
- if (v.size() <= 0) {
- return null;
- }
-
- Iterator<T> iterator = v.iterator();
- T min = iterator.next();
- while (iterator.hasNext()) {
- min = min(min, iterator.next());
- }
- return min;
- }
-
- public T max(Collection<T> v) {
- if (v.size() <= 0) {
- return null;
- }
-
- Iterator<T> iterator = v.iterator();
- T max = iterator.next();
- while (iterator.hasNext()) {
- max = max(max, iterator.next());
- }
- return max;
- }
-
- public T min(T a, T b) {
- return compare(a, b) <= 0 ? a : b;
- }
-
- public T max(T a, T b) {
- return compare(a, b) >= 0 ? a : b;
- }
-
- public boolean between(T v, T start, T end) {
- return compare(start, v) <= 0 && compare(v, end) <= 0;
- }
- }
-
- public static RecordComparator getRangeStartComparator(final IGTComparator comp) {
- return new RecordComparator(new ComparatorEx<ByteArray>() {
- @Override
- public int compare(ByteArray a, ByteArray b) {
- if (a.array() == null) {
- if (b.array() == null) {
- return 0;
- } else {
- return -1;
- }
- } else if (b.array() == null) {
- return 1;
- } else {
- return comp.compare(a, b);
- }
- }
- });
- }
-
- public static RecordComparator getRangeEndComparator(final IGTComparator comp) {
- return new RecordComparator(new ComparatorEx<ByteArray>() {
- @Override
- public int compare(ByteArray a, ByteArray b) {
- if (a.array() == null) {
- if (b.array() == null) {
- return 0;
- } else {
- return 1;
- }
- } else if (b.array() == null) {
- return -1;
- } else {
- return comp.compare(a, b);
- }
- }
- });
- }
-
- public static RecordComparator getRangeStartEndComparator(final IGTComparator comp) {
- return new AsymmetricRecordComparator(new ComparatorEx<ByteArray>() {
- @Override
- public int compare(ByteArray a, ByteArray b) {
- if (a.array() == null || b.array() == null) {
- return -1;
- } else {
- return comp.compare(a, b);
- }
- }
- });
- }
-
- private static class RecordComparator extends ComparatorEx<GTRecord> {
- final ComparatorEx<ByteArray> comparator;
-
- RecordComparator(ComparatorEx<ByteArray> byteComparator) {
- this.comparator = byteComparator;
- }
-
- @Override
- public int compare(GTRecord a, GTRecord b) {
- assert a.getInfo() == b.getInfo();
-
- int comp;
- ImmutableBitSet allColumns = a.getInfo().getAllColumns();
- for (int i = 0; i < allColumns.trueBitCount(); i++) {
- int c = allColumns.trueBitAt(i);
- comp = comparator.compare(a.get(c), b.get(c));
- if (comp != 0)
- return comp;
- }
- return 0; // equals
- }
- }
-
- /**
- * asymmetric means compare(a,b) > 0 does not cause compare(b,a) < 0
- * so min max functions will not be supported
- */
- private static class AsymmetricRecordComparator extends RecordComparator {
-
- AsymmetricRecordComparator(ComparatorEx<ByteArray> byteComparator) {
- super(byteComparator);
- }
-
- public GTRecord min(Collection<GTRecord> v) {
- throw new UnsupportedOperationException();
- }
-
- public GTRecord max(Collection<GTRecord> v) {
- throw new UnsupportedOperationException();
- }
-
- public GTRecord min(GTRecord a, GTRecord b) {
- throw new UnsupportedOperationException();
- }
-
- public GTRecord max(GTRecord a, GTRecord b) {
- throw new UnsupportedOperationException();
- }
-
- public boolean between(GTRecord v, GTRecord start, GTRecord end) {
- throw new UnsupportedOperationException();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
index 0904686..2e5dd12 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
@@ -20,9 +20,12 @@ package org.apache.kylin.cube.gridtable;
import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -120,16 +123,8 @@ public class CuboidToGridTableMapping {
return nDimensions + nMetrics;
}
- public int getDimensionCount() {
- return nDimensions;
- }
-
- public int getMetricsCount() {
- return nMetrics;
- }
-
public DataType[] getDataTypes() {
- return (DataType[]) gtDataTypes.toArray(new DataType[gtDataTypes.size()]);
+ return gtDataTypes.toArray(new DataType[gtDataTypes.size()]);
}
public ImmutableBitSet getPrimaryKey() {
@@ -137,7 +132,7 @@ public class CuboidToGridTableMapping {
}
public ImmutableBitSet[] getColumnBlocks() {
- return (ImmutableBitSet[]) gtColBlocks.toArray(new ImmutableBitSet[gtColBlocks.size()]);
+ return gtColBlocks.toArray(new ImmutableBitSet[gtColBlocks.size()]);
}
public int getIndexOf(TblColRef dimension) {
@@ -147,7 +142,7 @@ public class CuboidToGridTableMapping {
public int getIndexOf(FunctionDesc metric) {
Integer r = metrics2gt.get(metric);
- return r == null ? -1 : r.intValue();
+ return r == null ? -1 : r;
}
public List<TblColRef> getCuboidDimensionsInGTOrder() {
@@ -185,4 +180,47 @@ public class CuboidToGridTableMapping {
return result.isEmpty() ? Collections.<Integer, Integer> emptyMap() : result;
}
+ public ImmutableBitSet makeGridTableColumns(Set<TblColRef> dimensions) {
+ BitSet result = new BitSet();
+ for (TblColRef dim : dimensions) {
+ int idx = this.getIndexOf(dim);
+ if (idx >= 0)
+ result.set(idx);
+ }
+ return new ImmutableBitSet(result);
+ }
+
+ public ImmutableBitSet makeGridTableColumns(Collection<FunctionDesc> metrics) {
+ BitSet result = new BitSet();
+ for (FunctionDesc metric : metrics) {
+ int idx = this.getIndexOf(metric);
+ if (idx < 0)
+ throw new IllegalStateException(metric + " not found in " + this);
+ result.set(idx);
+ }
+ return new ImmutableBitSet(result);
+ }
+
+ public String[] makeAggrFuncs(Collection<FunctionDesc> metrics) {
+
+ //metrics are represented in ImmutableBitSet, which loses order information
+ //sort the aggrFuns to align with metrics natural order
+ List<FunctionDesc> metricList = Lists.newArrayList(metrics);
+ Collections.sort(metricList, new Comparator<FunctionDesc>() {
+ @Override
+ public int compare(FunctionDesc o1, FunctionDesc o2) {
+ int a = CuboidToGridTableMapping.this.getIndexOf(o1);
+ int b = CuboidToGridTableMapping.this.getIndexOf(o2);
+ return a - b;
+ }
+ });
+
+ String[] result = new String[metricList.size()];
+ int i = 0;
+ for (FunctionDesc metric : metricList) {
+ result[i++] = metric.getExpression();
+ }
+ return result;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparator.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparator.java
new file mode 100644
index 0000000..5360822
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparator.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.gridtable;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.gridtable.GTRecord;
+
+public class RecordComparator extends ComparatorEx<GTRecord> {
+ final public ComparatorEx<ByteArray> comparator;
+
+ RecordComparator(ComparatorEx<ByteArray> byteComparator) {
+ this.comparator = byteComparator;
+ }
+
+ @Override
+ public int compare(GTRecord a, GTRecord b) {
+ assert a.getInfo() == b.getInfo();
+
+ int comp;
+ ImmutableBitSet allColumns = a.getInfo().getAllColumns();
+ for (int i = 0; i < allColumns.trueBitCount(); i++) {
+ int c = allColumns.trueBitAt(i);
+ comp = comparator.compare(a.get(c), b.get(c));
+ if (comp != 0)
+ return comp;
+ }
+ return 0; // equals
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparators.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparators.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparators.java
new file mode 100644
index 0000000..43ea30b
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparators.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.gridtable;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.gridtable.IGTComparator;
+
+public class RecordComparators {
+
+ public static RecordComparator getRangeStartComparator(final IGTComparator comp) {
+ return new RecordComparator(new ComparatorEx<ByteArray>() {
+ @Override
+ public int compare(ByteArray a, ByteArray b) {
+ if (a.array() == null) {
+ if (b.array() == null) {
+ return 0;
+ } else {
+ return -1;
+ }
+ } else if (b.array() == null) {
+ return 1;
+ } else {
+ return comp.compare(a, b);
+ }
+ }
+ });
+ }
+
+ public static RecordComparator getRangeEndComparator(final IGTComparator comp) {
+ return new RecordComparator(new ComparatorEx<ByteArray>() {
+ @Override
+ public int compare(ByteArray a, ByteArray b) {
+ if (a.array() == null) {
+ if (b.array() == null) {
+ return 0;
+ } else {
+ return 1;
+ }
+ } else if (b.array() == null) {
+ return -1;
+ } else {
+ return comp.compare(a, b);
+ }
+ }
+ });
+ }
+
+ public static RecordComparator getRangeStartEndComparator(final IGTComparator comp) {
+ return new AsymmetricRecordComparator(new ComparatorEx<ByteArray>() {
+ @Override
+ public int compare(ByteArray a, ByteArray b) {
+ if (a.array() == null || b.array() == null) {
+ return -1;
+ } else {
+ return comp.compare(a, b);
+ }
+ }
+ });
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java
new file mode 100644
index 0000000..d938f2b
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.gridtable;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+public abstract class ScanRangePlannerBase {
+
+ //GT
+ protected TupleFilter gtFilter;
+ protected GTInfo gtInfo;
+ protected Pair<ByteArray, ByteArray> gtStartAndEnd;
+ protected TblColRef gtPartitionCol;
+ protected ImmutableBitSet gtDimensions;
+ protected ImmutableBitSet gtAggrGroups;
+ protected ImmutableBitSet gtAggrMetrics;
+ protected String[] gtAggrFuncs;
+ protected boolean isPartitionColUsingDatetimeEncoding = true;
+
+ protected RecordComparator rangeStartComparator;
+ protected RecordComparator rangeEndComparator;
+ protected RecordComparator rangeStartEndComparator;
+
+ public abstract GTScanRequest planScanRequest();
+
+ protected TupleFilter flattenToOrAndFilter(TupleFilter filter) {
+ if (filter == null)
+ return null;
+
+ TupleFilter flatFilter = filter.flatFilter();
+
+ // normalize to OR-AND filter
+ if (flatFilter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
+ LogicalTupleFilter f = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
+ f.addChild(flatFilter);
+ flatFilter = f;
+ }
+
+ if (flatFilter.getOperator() != TupleFilter.FilterOperatorEnum.OR)
+ throw new IllegalStateException();
+
+ return flatFilter;
+ }
+
+ protected List<Collection<ColumnRange>> translateToOrAndDimRanges(TupleFilter flatFilter) {
+ List<Collection<ColumnRange>> result = Lists.newArrayList();
+
+ if (flatFilter == null) {
+ result.add(Collections.<ColumnRange> emptyList());
+ return result;
+ }
+
+ for (TupleFilter andFilter : flatFilter.getChildren()) {
+ if (andFilter.getOperator() != TupleFilter.FilterOperatorEnum.AND)
+ throw new IllegalStateException("Filter should be AND instead of " + andFilter);
+
+ Collection<ColumnRange> andRanges = translateToAndDimRanges(andFilter.getChildren());
+ if (andRanges != null) {
+ result.add(andRanges);
+ }
+ }
+
+ return preEvaluateConstantConditions(result);
+ }
+
+ private Collection<ColumnRange> translateToAndDimRanges(List<? extends TupleFilter> andFilters) {
+ Map<TblColRef, ColumnRange> rangeMap = new HashMap<TblColRef, ColumnRange>();
+ for (TupleFilter filter : andFilters) {
+ if ((filter instanceof CompareTupleFilter) == false) {
+ if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) {
+ return null;
+ } else {
+ continue;
+ }
+ }
+
+ CompareTupleFilter comp = (CompareTupleFilter) filter;
+ if (comp.getColumn() == null) {
+ continue;
+ }
+
+ @SuppressWarnings("unchecked")
+ ColumnRange newRange = new ColumnRange(comp.getColumn(), (Set<ByteArray>) comp.getValues(), comp.getOperator());
+ ColumnRange existing = rangeMap.get(newRange.column);
+ if (existing == null) {
+ rangeMap.put(newRange.column, newRange);
+ } else {
+ existing.andMerge(newRange);
+ }
+ }
+ return rangeMap.values();
+ }
+
+ private List<Collection<ColumnRange>> preEvaluateConstantConditions(List<Collection<ColumnRange>> orAndRanges) {
+ boolean globalAlwaysTrue = false;
+ Iterator<Collection<ColumnRange>> iterator = orAndRanges.iterator();
+ while (iterator.hasNext()) {
+ Collection<ColumnRange> andRanges = iterator.next();
+ Iterator<ColumnRange> iterator2 = andRanges.iterator();
+ boolean hasAlwaysFalse = false;
+ while (iterator2.hasNext()) {
+ ColumnRange range = iterator2.next();
+ if (range.satisfyAll())
+ iterator2.remove();
+ else if (range.satisfyNone())
+ hasAlwaysFalse = true;
+ }
+ if (hasAlwaysFalse) {
+ iterator.remove();
+ } else if (andRanges.isEmpty()) {
+ globalAlwaysTrue = true;
+ break;
+ }
+ }
+ // return empty OR list means global false
+ // return an empty AND collection inside OR list means global true
+ if (globalAlwaysTrue) {
+ orAndRanges.clear();
+ orAndRanges.add(Collections.<ColumnRange> emptyList());
+ }
+ return orAndRanges;
+ }
+
+ public class ColumnRange {
+ public TblColRef column;
+ public ByteArray begin = ByteArray.EMPTY;
+ public ByteArray end = ByteArray.EMPTY;
+ public Set<ByteArray> valueSet;
+ public boolean isBoundryInclusive;
+
+ public ColumnRange(TblColRef column, Set<ByteArray> values, TupleFilter.FilterOperatorEnum op) {
+ this.column = column;
+
+ //TODO: the treatment is un-precise
+ if (op == TupleFilter.FilterOperatorEnum.EQ || op == TupleFilter.FilterOperatorEnum.IN || op == TupleFilter.FilterOperatorEnum.LTE || op == TupleFilter.FilterOperatorEnum.GTE) {
+ isBoundryInclusive = true;
+ }
+
+ switch (op) {
+ case EQ:
+ case IN:
+ valueSet = new HashSet<ByteArray>(values);
+ refreshBeginEndFromEquals();
+ break;
+ case LT:
+ case LTE:
+ end = rangeEndComparator.comparator.max(values);
+ break;
+ case GT:
+ case GTE:
+ begin = rangeStartComparator.comparator.min(values);
+ break;
+ case NEQ:
+ case NOTIN:
+ case ISNULL:
+ case ISNOTNULL:
+ // let Optiq filter it!
+ break;
+ default:
+ throw new UnsupportedOperationException(op.name());
+ }
+ }
+
+ void copy(TblColRef column, ByteArray beginValue, ByteArray endValue, Set<ByteArray> equalValues) {
+ this.column = column;
+ this.begin = beginValue;
+ this.end = endValue;
+ this.valueSet = equalValues;
+ }
+
+ private void refreshBeginEndFromEquals() {
+ if (valueSet.isEmpty()) {
+ begin = ByteArray.EMPTY;
+ end = ByteArray.EMPTY;
+ } else {
+ begin = rangeStartComparator.comparator.min(valueSet);
+ end = rangeEndComparator.comparator.max(valueSet);
+ }
+ }
+
+ public boolean satisfyAll() {
+ return begin.array() == null && end.array() == null; // the NEQ case
+ }
+
+ public boolean satisfyNone() {
+ if (valueSet != null) {
+ return valueSet.isEmpty();
+ } else if (begin.array() != null && end.array() != null) {
+ return gtInfo.getCodeSystem().getComparator().compare(begin, end) > 0;
+ } else {
+ return false;
+ }
+ }
+
+ public void andMerge(ColumnRange another) {
+ assert this.column.equals(another.column);
+
+ if (another.satisfyAll()) {
+ return;
+ }
+
+ if (this.satisfyAll()) {
+ copy(another.column, another.begin, another.end, another.valueSet);
+ return;
+ }
+
+ if (this.valueSet != null && another.valueSet != null) {
+ this.valueSet.retainAll(another.valueSet);
+ refreshBeginEndFromEquals();
+ return;
+ }
+
+ if (this.valueSet != null) {
+ this.valueSet = filter(this.valueSet, another.begin, another.end);
+ refreshBeginEndFromEquals();
+ return;
+ }
+
+ if (another.valueSet != null) {
+ this.valueSet = filter(another.valueSet, this.begin, this.end);
+ refreshBeginEndFromEquals();
+ return;
+ }
+
+ this.begin = rangeStartComparator.comparator.max(this.begin, another.begin);
+ this.end = rangeEndComparator.comparator.min(this.end, another.end);
+ this.isBoundryInclusive |= another.isBoundryInclusive;
+ }
+
+ private Set<ByteArray> filter(Set<ByteArray> equalValues, ByteArray beginValue, ByteArray endValue) {
+ Set<ByteArray> result = Sets.newHashSetWithExpectedSize(equalValues.size());
+ for (ByteArray v : equalValues) {
+ if (rangeStartEndComparator.comparator.compare(beginValue, v) <= 0 && rangeStartEndComparator.comparator.compare(v, endValue) <= 0) {
+ result.add(v);
+ }
+ }
+ return equalValues;
+ }
+
+ public String toString() {
+ if (valueSet == null) {
+ return column.getName() + " between " + begin + " and " + end;
+ } else {
+ return column.getName() + " in " + valueSet;
+ }
+ }
+
+ }
+
+ protected String makeReadable(ByteArray byteArray) {
+ if (byteArray == null) {
+ return null;
+ } else {
+ return byteArray.toReadableText();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
new file mode 100644
index 0000000..e31111d
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.gridtable;
+
+import java.nio.ByteBuffer;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.ISegment;
+import org.apache.kylin.dimension.AbstractDateDimEnc;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.metadata.datatype.DataType;
+
+public class SegmentGTStartAndEnd {
+ private ISegment segment;
+ private GTInfo info;
+
+ public SegmentGTStartAndEnd(ISegment segment, GTInfo info) {
+ this.segment = segment;
+ this.info = info;
+ }
+
+ public boolean isUsingDatetimeEncoding(int index) {
+ return info.getCodeSystem().getDimEnc(index) instanceof AbstractDateDimEnc;
+ }
+
+ public Pair<ByteArray, ByteArray> getSegmentStartAndEnd(int index) {
+ ByteArray start;
+ if (segment.getDateRangeStart() != Long.MIN_VALUE) {
+ start = encodeTime(segment.getDateRangeStart(), index, 1);
+ } else {
+ start = new ByteArray();
+ }
+
+ ByteArray end;
+ if (segment.getDateRangeEnd() != Long.MAX_VALUE) {
+ end = encodeTime(segment.getDateRangeEnd(), index, -1);
+ } else {
+ end = new ByteArray();
+ }
+ return Pair.newPair(start, end);
+
+ }
+
+ private ByteArray encodeTime(long ts, int index, int roundingFlag) {
+ String value;
+ DataType partitionColType = info.getColumnType(index);
+ if (partitionColType.isDate()) {
+ value = DateFormat.formatToDateStr(ts);
+ } else if (partitionColType.isDatetime() || partitionColType.isTimestamp()) {
+ value = DateFormat.formatToTimeWithoutMilliStr(ts);
+ } else if (partitionColType.isStringFamily()) {
+ String partitionDateFormat = segment.getModel().getPartitionDesc().getPartitionDateFormat();
+ if (StringUtils.isEmpty(partitionDateFormat))
+ partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN;
+ value = DateFormat.formatToDateStr(ts, partitionDateFormat);
+ } else {
+ throw new RuntimeException("Type " + partitionColType + " is not valid partition column type");
+ }
+
+ ByteBuffer buffer = ByteBuffer.allocate(info.getMaxColumnLength());
+ info.getCodeSystem().encodeColumnValue(index, value, roundingFlag, buffer);
+
+ return ByteArray.copyOf(buffer.array(), 0, buffer.position());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
index 21da4ea..12a3593 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
@@ -31,8 +31,11 @@ import org.apache.kylin.cube.gridtable.CubeCodeSystem;
import org.apache.kylin.cube.gridtable.TrimmedCubeCodeSystem;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class GTInfo {
+ private static final Logger logger = LoggerFactory.getLogger(GTInfo.class);
public static Builder builder() {
return new Builder();
@@ -151,7 +154,7 @@ public class GTInfo {
if (!expected.equals(ref))
throw new IllegalArgumentException();
}
-
+
void validate() {
if (codeSystem == null)
throw new IllegalStateException();
@@ -271,7 +274,7 @@ public class GTInfo {
public IGTCodeSystem getCodeSystem() {
return codeSystem;
}
-
+
public int getMaxLength() {
int ret = 0;
for (int i = 0; i < colAll.trueBitCount(); i++) {
@@ -291,7 +294,7 @@ public class GTInfo {
BytesUtil.writeAsciiString(GTSampleCodeSystem.class.getCanonicalName(), out);
GTSampleCodeSystem.serializer.serialize((GTSampleCodeSystem) value.codeSystem, out);
} else {
- throw new IllegalArgumentException("Can't recognize code system " + value.codeSystem.getClass());
+ BytesUtil.writeAsciiString(value.codeSystem.getClass().getCanonicalName(), out);
}
BytesUtil.writeUTFString(value.tableName, out);
@@ -317,7 +320,11 @@ public class GTInfo {
} else if (GTSampleCodeSystem.class.getCanonicalName().equals(codeSystemType)) {
codeSystem = GTSampleCodeSystem.serializer.deserialize(in);
} else {
- throw new IllegalArgumentException("Can't recognize code system " + codeSystemType);
+ try {
+ codeSystem = (IGTCodeSystem) Class.forName(codeSystemType).newInstance();
+ } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
}
String newTableName = BytesUtil.readUTFString(in);
@@ -349,5 +356,4 @@ public class GTInfo {
}
};
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index e2bac3d..4cfba1b 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -54,8 +54,8 @@ public class GTScanRequest {
// optional aggregation
private ImmutableBitSet aggrGroupBy;
private ImmutableBitSet aggrMetrics;
- private String[] aggrMetricsFuncs;
-
+ private String[] aggrMetricsFuncs;//
+
// hint to storage behavior
private boolean allowStorageAggregation;
private double aggCacheMemThreshold;
@@ -204,7 +204,7 @@ public class GTScanRequest {
//TODO BUG? select sum() from fact, no aggr by
public boolean hasAggregation() {
- return aggrGroupBy != null && aggrMetrics != null && aggrMetricsFuncs != null;
+ return !aggrGroupBy.isEmpty() || !aggrMetrics.isEmpty();
}
public GTInfo getInfo() {
@@ -284,16 +284,6 @@ public class GTScanRequest {
this.storagePushDownLimit = limit;
}
- public List<Integer> getRequiredMeasures() {
- List<Integer> measures = Lists.newArrayList();
- int numDim = info.getPrimaryKey().trueBitCount();
- for (int i = 0; i < aggrMetrics.trueBitCount(); i++) {
- int index = aggrMetrics.trueBitAt(i);
- measures.add(index - numDim);
- }
- return measures;
- }
-
@Override
public String toString() {
return "GTScanRequest [range=" + ranges + ", columns=" + columns + ", filterPushDown=" + filterPushDown + ", aggrGroupBy=" + aggrGroupBy + ", aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" + Arrays.toString(aggrMetricsFuncs) + "]";
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
index 49ec759..c4390cd 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
@@ -18,6 +18,7 @@
package org.apache.kylin.gridtable;
+import java.util.BitSet;
import java.util.List;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -92,6 +93,18 @@ public class GTScanRequestBuilder {
}
public GTScanRequest createGTScanRequest() {
+ if (aggrGroupBy == null) {
+ aggrGroupBy = new ImmutableBitSet(new BitSet());
+ }
+
+ if (aggrMetrics == null) {
+ aggrMetrics = new ImmutableBitSet(new BitSet());
+ }
+
+ if (aggrMetricsFuncs == null) {
+ aggrMetricsFuncs = new String[0];
+ }
+
return new GTScanRequest(info, ranges, dimensions, aggrGroupBy, aggrMetrics, aggrMetricsFuncs, filterPushDown, allowStorageAggregation, aggCacheMemThreshold, storageScanRowNumThreshold, storagePushDownLimit);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
index 5be91be..bb7503a 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
@@ -24,7 +24,7 @@ import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.Iterator;
-import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.ISegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,9 +34,9 @@ public class ScannerWorker {
private static final Logger logger = LoggerFactory.getLogger(ScannerWorker.class);
private IGTScanner internal = null;
- public ScannerWorker(CubeSegment cubeSeg, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage) {
+ public ScannerWorker(ISegment segment, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage) {
if (scanRequest == null) {
- logger.info("Segment {} will be skipped", cubeSeg);
+ logger.info("Segment {} will be skipped", segment);
internal = new EmptyGTScanner(0);
return;
}
@@ -44,7 +44,7 @@ public class ScannerWorker {
final GTInfo info = scanRequest.getInfo();
try {
- IGTStorage rpc = (IGTStorage) Class.forName(gtStorage).getConstructor(CubeSegment.class, Cuboid.class, GTInfo.class).newInstance(cubeSeg, cuboid, info); // default behavior
+ IGTStorage rpc = (IGTStorage) Class.forName(gtStorage).getConstructor(ISegment.class, Cuboid.class, GTInfo.class).newInstance(segment, cuboid, info); // default behavior
internal = rpc.getGTScanner(scanRequest);
} catch (IOException | InstantiationException | InvocationTargetException | IllegalAccessException | ClassNotFoundException | NoSuchMethodException e) {
throw new RuntimeException(e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index 2bbf6b7..7b6d3fa 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -132,7 +132,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
LogicalTupleFilter filter = and(timeComp4, ageComp1);
CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());
+ assertEquals(0, r.size());
}
{
LogicalTupleFilter filter = and(timeComp5, ageComp1);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index da2cf98..1f883bd 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -32,6 +32,7 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.apache.kylin.metadata.realization.RealizationType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -206,6 +207,13 @@ class ProjectL2Cache {
} else {
logger.warn("Realization '" + entry + "' defined under project '" + project + "' is not found");
}
+
+ //check if there's raw table parasite
+ //TODO: ugly impl here
+ IRealization parasite = registry.getRealization(RealizationType.INVERTED_INDEX, entry.getRealization());
+ if (parasite != null) {
+ projectCache.realizations.add(parasite);
+ }
}
for (IRealization realization : projectCache.realizations) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index aa90fc5..1eee1e7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -44,7 +44,6 @@ public class SQLDigest {
public Collection<FunctionDesc> aggregations;
public Collection<MeasureDesc> sortMeasures;
public Collection<OrderEnum> sortOrders;
- private boolean isRawQuery = false;
public SQLDigest(String factTable, TupleFilter filter, Collection<JoinDesc> joinDescs, Collection<TblColRef> allColumns, //
Collection<TblColRef> groupbyColumns, Collection<TblColRef> filterColumns, Collection<TblColRef> aggregatedColumns, Collection<FunctionDesc> aggregateFunnc, Collection<MeasureDesc> sortMeasures, Collection<OrderEnum> sortOrders) {
@@ -58,11 +57,12 @@ public class SQLDigest {
this.aggregations = aggregateFunnc;
this.sortMeasures = sortMeasures;
this.sortOrders = sortOrders;
- this.isRawQuery = this.groupbyColumns.isEmpty() && this.metricColumns.isEmpty();
}
public boolean isRawQuery() {
- return isRawQuery;
+ return this.groupbyColumns.isEmpty() && // select a group by a -> not raw
+ this.aggregations.isEmpty(); // has aggr -> not raw
+ //the reason to choose aggregations rather than metricColumns is because the former is set earlier at implOLAP
}
@Override