You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/11/23 07:34:27 UTC
[1/2] incubator-kylin git commit: KYLIN-942 code review
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-942-review d55da4073 -> c18f67811
KYLIN-942 code review
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/78b5712d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/78b5712d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/78b5712d
Branch: refs/heads/KYLIN-942-review
Commit: 78b5712df0a1057d8d9e4f8364e2475f52edd70d
Parents: d55da40
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Nov 23 13:14:51 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Nov 23 13:14:51 2015 +0800
----------------------------------------------------------------------
.../storage/hbase/cube/v1/CubeStorageQuery.java | 2 +-
.../cube/v1/SerializedHBaseTupleIterator.java | 22 +++++++++++++-------
.../storage/hbase/cube/v2/CubeStorageQuery.java | 22 ++++++++++++++------
.../storage/hbase/steps/RowValueDecoder.java | 12 +++++++++++
4 files changed, 43 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78b5712d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 8379572..4d34943 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -152,7 +152,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
HConnection conn = HBaseConnection.get(context.getConnUrl());
// notice we're passing filterD down to storage instead of flatFilter
- return new SerializedHBaseTupleIterator(conn, scans, cubeInstance, dimensionsD, filterD, groupsCopD, topNCol, valueDecoders, context, returnTupleInfo);
+ return new SerializedHBaseTupleIterator(conn, scans, cubeInstance, dimensionsD, filterD, groupsCopD, valueDecoders, context, returnTupleInfo);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78b5712d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
index 831cadb..0983689 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.client.HConnection;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
@@ -58,7 +59,7 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
private ITuple next;
public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
- Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, TblColRef topNCol, List<RowValueDecoder> rowValueDecoders, //
+ Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, List<RowValueDecoder> rowValueDecoders, //
StorageContext context, TupleInfo returnTupleInfo) {
this.context = context;
@@ -67,14 +68,9 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
this.segmentIteratorList = new ArrayList<CubeSegmentTupleIterator>(segmentKeyRanges.size());
Map<CubeSegment, List<HBaseKeyRange>> rangesMap = makeRangesMap(segmentKeyRanges);
- boolean useTopN = topNCol != null;
+
for (Map.Entry<CubeSegment, List<HBaseKeyRange>> entry : rangesMap.entrySet()) {
- CubeSegmentTupleIterator segIter;
- if (useTopN)
- segIter = new CubeSegmentTopNTupleIterator(entry.getKey(), entry.getValue(), conn, dimensions, filter, groupBy, topNCol, rowValueDecoders, context, returnTupleInfo);
- else
- segIter = new CubeSegmentTupleIterator(entry.getKey(), entry.getValue(), conn, dimensions, filter, groupBy, rowValueDecoders, context, returnTupleInfo);
- this.segmentIteratorList.add(segIter);
+ this.segmentIteratorList.add(newCubeSegmentTupleIterator(entry.getKey(), entry.getValue(), conn, dimensions, filter, groupBy, rowValueDecoders, context, returnTupleInfo));
}
this.segmentIteratorIterator = this.segmentIteratorList.iterator();
@@ -85,6 +81,16 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
}
}
+ private CubeSegmentTupleIterator newCubeSegmentTupleIterator(CubeSegment seg, List<HBaseKeyRange> keyRange, HConnection conn, Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, List<RowValueDecoder> rowValueDecoders, StorageContext context2, TupleInfo returnTupleInfo) {
+ MeasureDesc topN = RowValueDecoder.findTopN(rowValueDecoders);
+ if (topN != null) {
+ TblColRef topNCol = topN.getFunction().getTopNLiteralColumn();
+ return new CubeSegmentTopNTupleIterator(seg, keyRange, conn, dimensions, filter, groupBy, topNCol, rowValueDecoders, context, returnTupleInfo);
+ } else {
+ return new CubeSegmentTupleIterator(seg, keyRange, conn, dimensions, filter, groupBy, rowValueDecoders, context, returnTupleInfo);
+ }
+ }
+
private Map<CubeSegment, List<HBaseKeyRange>> makeRangesMap(List<HBaseKeyRange> segmentKeyRanges) {
Map<CubeSegment, List<HBaseKeyRange>> map = Maps.newHashMap();
for (HBaseKeyRange range : segmentKeyRanges) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78b5712d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index 258e20e..0c8c3bd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -37,7 +37,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
-@SuppressWarnings("unused")
public class CubeStorageQuery implements ICachableStorageQuery {
private static final Logger logger = LoggerFactory.getLogger(CubeStorageQuery.class);
@@ -113,10 +112,22 @@ public class CubeStorageQuery implements ICachableStorageQuery {
if (scanners.isEmpty())
return ITupleIterator.EMPTY_TUPLE_ITERATOR;
+ return newSequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context);
+ }
+
+ private ITupleIterator newSequentialCubeTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid, Set<TblColRef> dimensionsD, Set<FunctionDesc> metrics, TupleInfo returnTupleInfo, StorageContext context) {
+ TblColRef topNCol = null;
+ for (FunctionDesc func : metrics) {
+ if (func.isTopN()) {
+ topNCol = func.getTopNLiteralColumn();
+ break;
+ }
+ }
+
if (topNCol != null)
return new SequentialCubeTopNTupleIterator(scanners, cuboid, dimensionsD, topNCol, metrics, returnTupleInfo, context);
-
- return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context);
+ else
+ return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context);
}
private void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
@@ -132,7 +143,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
if (sqlDigest.metricColumns.contains(column)) {
continue;
}
-
+
dimensions.add(column);
}
}
@@ -382,7 +393,6 @@ public class CubeStorageQuery implements ICachableStorageQuery {
return false;
}
-
private void checkAndRewriteTopN(SQLDigest sqlDigest) {
FunctionDesc topnFunc = null;
TblColRef topnLiteralCol = null;
@@ -393,7 +403,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
topnLiteralCol = func.getTopNLiteralColumn();
}
}
-
+
// if TopN is not involved
if (topnFunc == null)
return;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78b5712d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
index 7c1a19f..50c2fac 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
@@ -140,4 +140,16 @@ public class RowValueDecoder implements Cloneable {
return false;
}
+ public static MeasureDesc findTopN(Collection<RowValueDecoder> rowValueDecoders) {
+ for (RowValueDecoder decoder : rowValueDecoders) {
+ for (int i = decoder.projectionIndex.nextSetBit(0); i >= 0; i = decoder.projectionIndex.nextSetBit(i + 1)) {
+ MeasureDesc measure = decoder.measures[i];
+ FunctionDesc func = measure.getFunction();
+ if (func.isTopN())
+ return measure;
+ }
+ }
+ return null;
+ }
+
}
[2/2] incubator-kylin git commit: KYLIN-942 code review
Posted by li...@apache.org.
KYLIN-942 code review
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c18f6781
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c18f6781
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c18f6781
Branch: refs/heads/KYLIN-942-review
Commit: c18f678114554d95c71da308ebe654731a662b5b
Parents: 78b5712
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Nov 23 14:34:03 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Nov 23 14:34:03 2015 +0800
----------------------------------------------------------------------
.../kylin/cube/CubeCapabilityChecker.java | 8 +-
.../cube/inmemcubing/InMemCubeBuilder.java | 15 +-
.../InMemCubeBuilderInputConverter.java | 14 +-
.../cube/inmemcubing/InMemCubeBuilderUtils.java | 10 +-
.../cube/inmemcubing/InMemCubeBuilderTest.java | 12 +-
.../engine/mr/steps/BaseCuboidMapperBase.java | 24 +--
.../localmeta/cube/test_kylin_cube_topn.json | 10 --
.../cube/test_kylin_cube_topn_left_join.json | 10 --
.../cube_desc/test_kylin_cube_topn_desc.json | 148 ------------------
.../test_kylin_cube_topn_left_join_desc.json | 149 -------------------
.../test_kylin_cube_without_slr_desc.json | 11 +-
...t_kylin_cube_without_slr_left_join_desc.json | 11 +-
12 files changed, 59 insertions(+), 363 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c18f6781/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 16353b3..343bf11 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -132,15 +132,15 @@ public class CubeCapabilityChecker {
for (MeasureDesc measure : cubeDesc.getMeasures()) {
if (measure.getFunction().isTopN()) {
List<TblColRef> cols = measure.getFunction().getParameter().getColRefs();
- TblColRef displayCol = cols.get(cols.size() - 1);
- if (digest.groupbyColumns.contains(displayCol)) {
- dimensionColumnsCopy.remove(displayCol);
+ TblColRef literalCol = cols.get(cols.size() - 1);
+ if (digest.groupbyColumns.contains(literalCol)) {
+ dimensionColumnsCopy.remove(literalCol);
if (isMatchedWithDimensions(dimensionColumnsCopy, cube)) {
if (measure.getFunction().isTopNCompatibleSum(onlyFunction)) {
return true;
}
}
- dimensionColumnsCopy.add(displayCol);
+ dimensionColumnsCopy.add(literalCol);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c18f6781/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index a179d70..e9d940a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -17,7 +17,12 @@
package org.apache.kylin.cube.inmemcubing;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
@@ -25,7 +30,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.common.util.Pair;
@@ -43,8 +47,6 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GridTable;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.metadata.measure.DoubleMutable;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -59,14 +61,12 @@ import com.google.common.collect.Lists;
public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
private static Logger logger = LoggerFactory.getLogger(InMemCubeBuilder.class);
- private static final LongMutable ONE = new LongMutable(1l);
static final double BASE_CUBOID_CACHE_OVERSIZE_FACTOR = 0.1;
private final CuboidScheduler cuboidScheduler;
private final long baseCuboidId;
private final int totalCuboidCount;
private final CubeJoinedFlatTableDesc intermediateTableDesc;
- private final MeasureCodec measureCodec;
private final String[] metricsAggrFuncs;
private final MeasureDesc[] measureDescs;
private final int measureCount;
@@ -87,7 +87,6 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
this.totalCuboidCount = cuboidScheduler.getCuboidCount();
this.intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);
- this.measureCodec = new MeasureCodec(cubeDesc.getMeasures());
this.measureCount = cubeDesc.getMeasures().size();
this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);
@@ -510,7 +509,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
this.input = input;
this.record = new GTRecord(info);
this.inMemCubeBuilderInputConverter = new InMemCubeBuilderInputConverter(cubeDesc,
- InMemCubeBuilderUtils.createTopNDisplayColDictionaryMap(cubeDesc, intermediateTableDesc, dictionaryMap),
+ InMemCubeBuilderUtils.createTopNLiteralColDictionaryMap(cubeDesc, intermediateTableDesc, dictionaryMap),
info);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c18f6781/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
index d9099ce..69a9fc9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
@@ -47,18 +47,18 @@ public class InMemCubeBuilderInputConverter {
private final MeasureCodec measureCodec;
private final int measureCount;
private final ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- private final Map<Integer, Dictionary<String>> topNDisplayColDictMap;
+ private final Map<Integer, Dictionary<String>> topNLiteralColDictMap;
private final GTInfo gtInfo;
- public InMemCubeBuilderInputConverter(CubeDesc cubeDesc, Map<Integer, Dictionary<String>> topNDisplayColDictMap, GTInfo gtInfo) {
+ public InMemCubeBuilderInputConverter(CubeDesc cubeDesc, Map<Integer, Dictionary<String>> topNLiteralColDictMap, GTInfo gtInfo) {
this.cubeDesc = cubeDesc;
this.gtInfo = gtInfo;
this.intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);
this.measureCount = cubeDesc.getMeasures().size();
this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);
this.measureCodec = new MeasureCodec(cubeDesc.getMeasures());
- this.topNDisplayColDictMap = Preconditions.checkNotNull(topNDisplayColDictMap, "topNDisplayColDictMap cannot be null");
+ this.topNLiteralColDictMap = Preconditions.checkNotNull(topNLiteralColDictMap, "topNLiteralColDictMap cannot be null");
}
public final GTRecord convert(List<String> row) {
@@ -102,13 +102,13 @@ public class InMemCubeBuilderInputConverter {
} else if (function.isTopN()) {
// encode the key column with dict, and get the counter column;
int keyColIndex = flatTableIdx[flatTableIdx.length - 1];
- Dictionary<String> displayColDict = topNDisplayColDictMap.get(keyColIndex);
- int keyColEncoded = displayColDict.getIdFromValue(row.get(keyColIndex));
+ Dictionary<String> literalColDict = topNLiteralColDictMap.get(keyColIndex);
+ int keyColEncoded = literalColDict.getIdFromValue(row.get(keyColIndex));
valueBuf.clear();
- valueBuf.putInt(displayColDict.getSizeOfId());
+ valueBuf.putInt(literalColDict.getSizeOfId());
valueBuf.putInt(keyColEncoded);
if (flatTableIdx.length == 1) {
- // only displayCol, use 1.0 as counter
+ // only literalCol, use 1.0 as counter
valueBuf.putDouble(1.0);
} else {
// get the counter column value
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c18f6781/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
index f0ee372..9d819a4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
@@ -35,19 +35,19 @@ import java.util.Map;
*/
public final class InMemCubeBuilderUtils {
- public static final HashMap<Integer, Dictionary<String>> createTopNDisplayColDictionaryMap(CubeDesc cubeDesc, CubeJoinedFlatTableDesc intermediateTableDesc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
+ public static final HashMap<Integer, Dictionary<String>> createTopNLiteralColDictionaryMap(CubeDesc cubeDesc, CubeJoinedFlatTableDesc intermediateTableDesc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
HashMap<Integer, Dictionary<String>> result = Maps.newHashMap();
for (int measureIdx = 0; measureIdx < cubeDesc.getMeasures().size(); measureIdx++) {
MeasureDesc measureDesc = cubeDesc.getMeasures().get(measureIdx);
FunctionDesc func = measureDesc.getFunction();
if (func.isTopN()) {
int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[measureIdx];
- int displayColIdx = flatTableIdx[flatTableIdx.length - 1];
- TblColRef displayCol = func.getParameter().getColRefs().get(flatTableIdx.length - 1);
+ int literalColIdx = flatTableIdx[flatTableIdx.length - 1];
+ TblColRef literalCol = func.getTopNLiteralColumn();
@SuppressWarnings("unchecked")
- Dictionary<String> dictionary = (Dictionary<String>) dictionaryMap.get(displayCol);
+ Dictionary<String> dictionary = (Dictionary<String>) dictionaryMap.get(literalCol);
//Preconditions.checkNotNull(dictionary);//FIXME disable check since dictionary is null when building empty segment
- result.put(displayColIdx, dictionary);
+ result.put(literalColIdx, dictionary);
}
}
return result;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c18f6781/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
index e3fb30e..f853b08 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
@@ -188,13 +188,13 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
FunctionDesc func = measureDesc.getFunction();
if (func.isTopN()) {
int[] flatTableIdx = flatTableDesc.getMeasureColumnIndexes()[measureIdx];
- int displayColIdx = flatTableIdx[flatTableIdx.length - 1];
- TblColRef displayCol = func.getParameter().getColRefs().get(flatTableIdx.length - 1);
- logger.info("Building dictionary for " + displayCol);
- List<byte[]> valueList = readValueList(flatTable, nColumns, displayColIdx);
- Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(displayCol.getType(), new IterableDictionaryValueEnumerator(valueList));
+ int literalColIdx = flatTableIdx[flatTableIdx.length - 1];
+ TblColRef literalCol = func.getTopNLiteralColumn();
+ logger.info("Building dictionary for " + literalCol);
+ List<byte[]> valueList = readValueList(flatTable, nColumns, literalColIdx);
+ Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(literalCol.getType(), new IterableDictionaryValueEnumerator(valueList));
- result.put(displayCol, dict);
+ result.put(literalCol, dict);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c18f6781/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index ed1fd4a..44311c2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -57,7 +57,7 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
private Text outputKey = new Text();
private Text outputValue = new Text();
private ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- private Map<Integer, Dictionary<String>> topNDisplayColDictMap;
+ private Map<Integer, Dictionary<String>> topNLiteralColDictMap;
@Override
protected void setup(Context context) throws IOException {
@@ -92,21 +92,21 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
int colCount = cubeDesc.getRowkey().getRowKeyColumns().length;
keyBytesBuf = new byte[colCount][];
- initTopNDisplayColDictionaryMap();
+ initTopNLiteralColDictionaryMap();
initNullBytes();
}
- private void initTopNDisplayColDictionaryMap() {
- topNDisplayColDictMap = Maps.newHashMap();
+ private void initTopNLiteralColDictionaryMap() {
+ topNLiteralColDictMap = Maps.newHashMap();
for (int measureIdx = 0; measureIdx < measures.length; measureIdx++) {
MeasureDesc measureDesc = cubeDesc.getMeasures().get(measureIdx);
FunctionDesc func = measureDesc.getFunction();
if (func.isTopN()) {
int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[measureIdx];
- int displayColIdx = flatTableIdx[flatTableIdx.length - 1];
- TblColRef displayCol = func.getParameter().getColRefs().get(flatTableIdx.length - 1);
- Dictionary<String> dictionary = (Dictionary<String>)cubeSegment.getDictionary(displayCol);
- topNDisplayColDictMap.put(displayColIdx, dictionary);
+ int literalColIdx = flatTableIdx[flatTableIdx.length - 1];
+ TblColRef literalCol = func.getTopNLiteralColumn();
+ Dictionary<String> dictionary = (Dictionary<String>)cubeSegment.getDictionary(literalCol);
+ topNLiteralColDictMap.put(literalColIdx, dictionary);
}
}
}
@@ -174,13 +174,13 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
else if(func.isTopN()) {
// encode the key column with dict, and get the counter column;
int keyColIndex = flatTableIdx[flatTableIdx.length - 1];
- Dictionary<String> displayColDict = topNDisplayColDictMap.get(keyColIndex);
- int keyColEncoded = displayColDict.getIdFromValue(Bytes.toString(splitBuffers[keyColIndex].value));
+ Dictionary<String> literalColDict = topNLiteralColDictMap.get(keyColIndex);
+ int keyColEncoded = literalColDict.getIdFromValue(Bytes.toString(splitBuffers[keyColIndex].value));
valueBuf.clear();
- valueBuf.putInt(displayColDict.getSizeOfId());
+ valueBuf.putInt(literalColDict.getSizeOfId());
valueBuf.putInt(keyColEncoded);
if (flatTableIdx.length == 1) {
- // only displayCol, use 1.0 as counter
+ // only literalCol, use 1.0 as counter
valueBuf.putDouble(1.0);
} else {
// get the counter column value
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c18f6781/examples/test_case_data/localmeta/cube/test_kylin_cube_topn.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_topn.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_topn.json
deleted file mode 100644
index 903fc15..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_topn.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "uuid" : "33354455-a33e-4b69-83dd-0bb8b1f8c53b",
- "last_modified" : 0,
- "name" : "test_kylin_cube_topn",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_topn_desc",
- "segments" : [ ],
- "create_time" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c18f6781/examples/test_case_data/localmeta/cube/test_kylin_cube_topn_left_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_topn_left_join.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_topn_left_join.json
deleted file mode 100644
index 6f57561..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_topn_left_join.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "uuid" : "44454455-a33e-4b69-83dd-0bb8b1f8c53b",
- "last_modified" : 0,
- "name" : "test_kylin_cube_topn_left_join",
- "owner" : null,
- "version" : null,
- "descriptor" : "test_kylin_cube_topn_left_join_desc",
- "segments" : [ ],
- "create_time" : null
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c18f6781/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_desc.json
deleted file mode 100644
index fddbb10..0000000
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_desc.json
+++ /dev/null
@@ -1,148 +0,0 @@
-{
- "uuid": "4334a905-1fc6-4f67-985c-38fa5aeafd92",
- "name": "test_kylin_cube_topn_desc",
- "description": null,
- "dimensions": [
- {
- "id": 0,
- "name": "CAL_DT",
- "table": "EDW.TEST_CAL_DT",
- "column": null,
- "derived": [
- "WEEK_BEG_DT"
- ],
- "hierarchy": false
- }
- ],
- "measures": [
- {
- "id": 1,
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 2,
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 3,
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 4,
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 5,
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 6,
- "name": "TOP_SELLER",
- "function": {
- "expression": "TOP_N",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "displaycolumn": "SELLER_ID"
- },
- "returntype": "topn(100)"
- },
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "cal_dt",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- }
- ],
- "aggregation_groups": [
- [
- "cal_dt"
- ]
- ]
- },
- "last_modified": 1422435345330,
- "model_name": "test_kylin_inner_join_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- }, {
- "name": "f2",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "top_seller"
- ]
- }
- ]
- }
- ]
- },
- "notify_list": null,
- "engine_type": 2,
- "storage_type": 2
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c18f6781/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_left_join_desc.json
deleted file mode 100644
index 6aecaae..0000000
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_topn_left_join_desc.json
+++ /dev/null
@@ -1,149 +0,0 @@
-{
- "uuid": "5445a905-1fc6-4f67-985c-38fa5aeafd92",
- "name": "test_kylin_cube_topn_left_join_desc",
- "description": null,
- "dimensions": [
- {
- "id": 0,
- "name": "CAL_DT",
- "table": "EDW.TEST_CAL_DT",
- "column": null,
- "derived": [
- "WEEK_BEG_DT"
- ],
- "hierarchy": false
- }
- ],
- "measures": [
- {
- "id": 1,
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 2,
- "name": "GMV_MIN",
- "function": {
- "expression": "MIN",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 3,
- "name": "GMV_MAX",
- "function": {
- "expression": "MAX",
- "parameter": {
- "type": "column",
- "value": "PRICE"
- },
- "returntype": "decimal(19,4)"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 4,
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 5,
- "name": "ITEM_COUNT_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "ITEM_COUNT"
- },
- "returntype": "bigint"
- },
- "dependent_measure_ref": null
- },
- {
- "id": 6,
- "name": "TOP_SELLER",
- "function": {
- "expression": "TOP_N",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "displaycolumn": "SELLER_ID"
- },
- "returntype": "topn(100)"
- },
- "dependent_measure_ref": null
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "cal_dt",
- "length": 0,
- "dictionary": "true",
- "mandatory": false
- }
- ],
- "aggregation_groups": [
- [
- "cal_dt"
- ]
- ]
- },
- "last_modified": 1422435345330,
- "model_name": "test_kylin_left_join_model_desc",
- "null_string": null,
- "hbase_mapping": {
- "column_family": [
- {
- "name": "f1",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "gmv_sum",
- "gmv_min",
- "gmv_max",
- "trans_cnt",
- "item_count_sum"
- ]
- }
- ]
- },
- {
- "name": "f2",
- "columns": [
- {
- "qualifier": "m",
- "measure_refs": [
- "top_seller"
- ]
- }
- ]
- }
- ]
- },
- "notify_list": null,
- "engine_type": 2,
- "storage_type": 2
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c18f6781/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index 3f9957b..1e007c3 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -156,7 +156,11 @@
"expression": "COUNT_DISTINCT",
"parameter": {
"type": "column",
- "value": "LSTG_FORMAT_NAME,SELLER_ID"
+ "value": "LSTG_FORMAT_NAME",
+ "next_parameter": {
+ "type": "column",
+ "value": "SELLER_ID"
+ }
},
"returntype": "hllc(10)"
},
@@ -170,7 +174,10 @@
"parameter": {
"type": "column",
"value": "PRICE",
- "displaycolumn": "SELLER_ID"
+ "next_parameter": {
+ "type": "column",
+ "value": "SELLER_ID"
+ }
},
"returntype": "topn(100)"
},
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c18f6781/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 907e338..73a58f0 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -156,7 +156,11 @@
"expression": "COUNT_DISTINCT",
"parameter": {
"type": "column",
- "value": "LSTG_FORMAT_NAME,SELLER_ID"
+ "value": "LSTG_FORMAT_NAME",
+ "next_parameter": {
+ "type": "column",
+ "value": "SELLER_ID"
+ }
},
"returntype": "hllc(10)"
},
@@ -170,7 +174,10 @@
"parameter": {
"type": "column",
"value": "PRICE",
- "displaycolumn": "SELLER_ID"
+ "next_parameter": {
+ "type": "column",
+ "value": "SELLER_ID"
+ }
},
"returntype": "topn(100)"
},