You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/06/14 09:28:45 UTC
[kylin] 10/10: APACHE-KYLIN-3359 change tupleExpressionList to
tupleExpressionMap & keep dynamicCols and tupleExpressionMap independent in
GTScanRequest
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a9557a6ccd32e89985330398bca4e73a054fa5a6
Author: Zhong <nj...@apache.org>
AuthorDate: Thu May 24 16:22:28 2018 +0800
APACHE-KYLIN-3359 change tupleExpressionList to tupleExpressionMap & keep dynamicCols and tupleExpressionMap independent in GTScanRequest
---
.../kylin/cube/gridtable/ScanRangePlannerBase.java | 2 +-
.../apache/kylin/gridtable/GTFunctionScanner.java | 23 ++++---------
.../org/apache/kylin/gridtable/GTScanRequest.java | 38 ++++++++++++++--------
.../kylin/gridtable/GTScanRequestBuilder.java | 9 ++---
.../storage/gtrecord/CubeScanRangePlanner.java | 27 ++++++++-------
5 files changed, 52 insertions(+), 47 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java
index b127935..811d512 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java
@@ -64,7 +64,7 @@ public abstract class ScanRangePlannerBase {
protected ImmutableBitSet gtDynColumns;
protected ImmutableBitSet gtRtAggrMetrics;
- protected List<TupleExpression> tupleExpressionList;
+ protected Map<Integer, TupleExpression> tupleExpressionMap;
public abstract GTScanRequest planScanRequest();
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFunctionScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFunctionScanner.java
index 1c9d686..65466f1 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFunctionScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFunctionScanner.java
@@ -20,7 +20,7 @@ package org.apache.kylin.gridtable;
import java.io.IOException;
import java.util.Iterator;
-import java.util.List;
+import java.util.Map;
import java.util.NoSuchElementException;
import org.apache.kylin.common.util.ByteArray;
@@ -33,16 +33,12 @@ import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-
public class GTFunctionScanner implements IGTScanner {
private static final Logger logger = LoggerFactory.getLogger(GTFunctionScanner.class);
protected IGTScanner rawScanner;
- private final ImmutableBitSet dynamicCols;
private final ImmutableBitSet rtAggrMetrics;
- private final List<TupleExpression> tupleExpressionList;
+ private final Map<Integer, TupleExpression> tupleExpressionMap;
private final IEvaluatableTuple oneTuple; // avoid instance creation
private final IFilterCodeSystem<ByteArray> filterCodeSystem;
@@ -50,8 +46,7 @@ public class GTFunctionScanner implements IGTScanner {
protected GTFunctionScanner(IGTScanner rawScanner, GTScanRequest req) {
this.rawScanner = rawScanner;
- this.dynamicCols = req.getDynamicCols();
- this.tupleExpressionList = req.getTupleExpressionList();
+ this.tupleExpressionMap = req.getTupleExpressionMap();
this.rtAggrMetrics = req.getRtAggrMetrics();
this.oneTuple = new IEvaluatableTuple() {
@Override
@@ -111,14 +106,10 @@ public class GTFunctionScanner implements IGTScanner {
}
private void calculateDynamics() {
- List<Object> rtResult = Lists.transform(tupleExpressionList, new Function<TupleExpression, Object>() {
- @Override
- public Object apply(TupleExpression tupleExpr) {
- return tupleExpr.calculate(oneTuple, filterCodeSystem);
- }
- });
- for (int i = 0; i < dynamicCols.trueBitCount(); i++) {
- next.setValue(dynamicCols.trueBitAt(i), rtResult.get(i)); //
+ for (int c : tupleExpressionMap.keySet()) {
+ TupleExpression tupleExpr = tupleExpressionMap.get(c);
+ Object rtResult = tupleExpr.calculate(oneTuple, filterCodeSystem);
+ next.setValue(c, rtResult);
}
}
};
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 9c75603..e3a2234 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -23,6 +23,7 @@ import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.commons.io.IOUtils;
@@ -45,6 +46,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
public class GTScanRequest {
@@ -65,7 +67,7 @@ public class GTScanRequest {
private ImmutableBitSet rtAggrMetrics;
private ImmutableBitSet dynamicCols;
- private List<TupleExpression> tupleExpressionList;
+ private Map<Integer, TupleExpression> tupleExpressionMap;
// optional filtering
private TupleFilter filterPushDown;
@@ -92,7 +94,7 @@ public class GTScanRequest {
GTScanRequest(GTInfo info, List<GTScanRange> ranges, ImmutableBitSet dimensions, ImmutableBitSet aggrGroupBy, //
ImmutableBitSet aggrMetrics, String[] aggrMetricsFuncs, ImmutableBitSet rtAggrMetrics, //
- ImmutableBitSet dynamicCols, List<TupleExpression> tupleExpressionList, //
+ ImmutableBitSet dynamicCols, Map<Integer, TupleExpression> tupleExpressionMap, //
TupleFilter filterPushDown, TupleFilter havingFilterPushDown, //
boolean allowStorageAggregation, double aggCacheMemThreshold, int storageScanRowNumThreshold, //
int storagePushDownLimit, StorageLimitLevel storageLimitLevel, String storageBehavior, long startTime,
@@ -114,7 +116,7 @@ public class GTScanRequest {
this.rtAggrMetrics = rtAggrMetrics;
this.dynamicCols = dynamicCols;
- this.tupleExpressionList = tupleExpressionList;
+ this.tupleExpressionMap = tupleExpressionMap;
this.storageBehavior = storageBehavior;
this.startTime = startTime;
@@ -217,8 +219,8 @@ public class GTScanRequest {
result = new GTForwardingScanner(result);//need its check function
}
- if (dynamicCols != null && dynamicCols.cardinality() > 0) {
- logger.info("GTFunctionScanner will be used with expressions " + tupleExpressionList);
+ if (tupleExpressionMap != null && !tupleExpressionMap.isEmpty()) {
+ logger.info("GTFunctionScanner will be used with expressions " + tupleExpressionMap);
result = new GTFunctionScanner(result, this);
}
@@ -344,8 +346,8 @@ public class GTScanRequest {
return rtAggrMetrics;
}
- public List<TupleExpression> getTupleExpressionList() {
- return tupleExpressionList;
+ public Map<Integer, TupleExpression> getTupleExpressionMap() {
+ return tupleExpressionMap;
}
public boolean isAllowStorageAggregation() {
@@ -446,7 +448,11 @@ public class GTScanRequest {
// for dynamic related info
ImmutableBitSet.serializer.serialize(value.dynamicCols, out);
- for (TupleExpression tupleExpr : value.tupleExpressionList) {
+
+ BytesUtil.writeVInt(value.tupleExpressionMap.size(), out);
+ for (int c : value.tupleExpressionMap.keySet()) {
+ TupleExpression tupleExpr = value.tupleExpressionMap.get(c);
+ BytesUtil.writeVInt(c, out);
BytesUtil.writeByteArray(TupleExpressionSerializer.serialize(tupleExpr,
GTUtil.wrap(value.info.codeSystem.getComparator())), out);
}
@@ -495,17 +501,21 @@ public class GTScanRequest {
String storageBehavior = BytesUtil.readUTFString(in);
ImmutableBitSet aDynCols = ImmutableBitSet.serializer.deserialize(in);
- int nDynCols = aDynCols.cardinality();
- List<TupleExpression> sTupleExprList = Lists.newArrayListWithExpectedSize(nDynCols);
- for (int i = 0; i < nDynCols; i++) {
- sTupleExprList.add(TupleExpressionSerializer.deserialize(BytesUtil.readByteArray(in),
- GTUtil.wrap(sInfo.codeSystem.getComparator())));
+
+ int nTupleExprs = BytesUtil.readVInt(in);
+ Map<Integer, TupleExpression> sTupleExpressionMap = Maps.newHashMapWithExpectedSize(nTupleExprs);
+ for (int i = 0; i < nTupleExprs; i++) {
+ int sC = BytesUtil.readVInt(in);
+ TupleExpression sTupleExpr = TupleExpressionSerializer.deserialize(BytesUtil.readByteArray(in),
+ GTUtil.wrap(sInfo.codeSystem.getComparator()));
+ sTupleExpressionMap.put(sC, sTupleExpr);
}
ImmutableBitSet aRuntimeAggrMetrics = ImmutableBitSet.serializer.deserialize(in);
return new GTScanRequestBuilder().setInfo(sInfo).setRanges(sRanges).setDimensions(sColumns)
.setAggrGroupBy(sAggGroupBy).setAggrMetrics(sAggrMetrics).setAggrMetricsFuncs(sAggrMetricFuncs)
- .setRtAggrMetrics(aRuntimeAggrMetrics).setDynamicColumns(aDynCols).setExprsPushDown(sTupleExprList)
+ .setRtAggrMetrics(aRuntimeAggrMetrics).setDynamicColumns(aDynCols)
+ .setExprsPushDown(sTupleExpressionMap)
.setFilterPushDown(sGTFilter).setHavingFilterPushDown(sGTHavingFilter)
.setAllowStorageAggregation(sAllowPreAggr).setAggCacheMemThreshold(sAggrCacheGB)
.setStorageScanRowNumThreshold(storageScanRowNumThreshold)
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
index 7dd8846..94a89e6 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
@@ -20,13 +20,14 @@ package org.apache.kylin.gridtable;
import java.util.BitSet;
import java.util.List;
+import java.util.Map;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.expression.TupleExpression;
import org.apache.kylin.metadata.filter.TupleFilter;
-import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
public class GTScanRequestBuilder {
private GTInfo info;
@@ -39,7 +40,7 @@ public class GTScanRequestBuilder {
private String[] aggrMetricsFuncs = null;
private ImmutableBitSet dynamicColumns;
private ImmutableBitSet rtAggrMetrics;
- private List<TupleExpression> exprsPushDown;
+ private Map<Integer, TupleExpression> exprsPushDown;
private boolean allowStorageAggregation = true;
private double aggCacheMemThreshold = 0;
private int storageScanRowNumThreshold = Integer.MAX_VALUE;// storage should terminate itself when $storageScanRowNumThreshold cuboid rows are scanned, and throw exception.
@@ -69,7 +70,7 @@ public class GTScanRequestBuilder {
return this;
}
- public GTScanRequestBuilder setExprsPushDown(List<TupleExpression> exprsPushDown) {
+ public GTScanRequestBuilder setExprsPushDown(Map<Integer, TupleExpression> exprsPushDown) {
this.exprsPushDown = exprsPushDown;
return this;
}
@@ -166,7 +167,7 @@ public class GTScanRequestBuilder {
}
if (exprsPushDown == null) {
- exprsPushDown = Lists.newArrayList();
+ exprsPushDown = Maps.newHashMap();
}
if (storageBehavior == null) {
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
index f99c868..5f86a45 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
@@ -114,24 +114,27 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase {
// for dynamic cols, which are as appended columns to GTInfo
BitSet tmpGtDynCols = new BitSet();
- tmpGtDynCols.or(mapping.makeGridTableColumns(Sets.newHashSet(dynGroupsDims)).mutable());
- tmpGtDynCols.or(mapping.makeGridTableColumns(dynFuncs).mutable());
- this.gtDynColumns = new ImmutableBitSet(tmpGtDynCols);
- this.tupleExpressionList = Lists.newArrayListWithExpectedSize(dynGroupExprs.size() + dynFuncs.size());
+ this.tupleExpressionMap = Maps.newHashMap();
// for dynamic dimensions
- for (TupleExpression rtGroupExpr : dynGroupExprs) {
- this.tupleExpressionList
- .add(GTUtil.convertFilterColumnsAndConstants(rtGroupExpr, gtInfo, mapping, groupByPushDown));
+ for (int i = 0; i < dynGroupsDims.size(); i++) {
+ int c = mapping.getIndexOf(dynGroupsDims.get(i));
+ tmpGtDynCols.set(c);
+ TupleExpression tupleExpr = GTUtil.convertFilterColumnsAndConstants(dynGroupExprs.get(i), gtInfo, mapping,
+ groupByPushDown);
+ this.tupleExpressionMap.put(c, tupleExpr);
}
// for dynamic measures
Set<FunctionDesc> tmpRtAggrMetrics = Sets.newHashSet();
- for (DynamicFunctionDesc rtFunc : dynFuncs) {
- tmpRtAggrMetrics.addAll(rtFunc.getRuntimeFuncs());
- this.tupleExpressionList.add(GTUtil.convertFilterColumnsAndConstants(rtFunc.getTupleExpression(), gtInfo,
- mapping, rtFunc.getRuntimeFuncMap(), groupByPushDown));
+ for (DynamicFunctionDesc dynFunc : dynFuncs) {
+ tmpRtAggrMetrics.addAll(dynFunc.getRuntimeFuncs());
+ int c = mapping.getIndexOf(dynFunc);
+ tmpGtDynCols.set(c);
+ this.tupleExpressionMap.put(c, GTUtil.convertFilterColumnsAndConstants(dynFunc.getTupleExpression(), gtInfo,
+ mapping, dynFunc.getRuntimeFuncMap(), groupByPushDown));
}
+ this.gtDynColumns = new ImmutableBitSet(tmpGtDynCols);
this.gtRtAggrMetrics = mapping.makeGridTableColumns(tmpRtAggrMetrics);
if (cubeSegment.getModel().getPartitionDesc().isPartitioned()) {
@@ -179,7 +182,7 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase {
.setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs)
.setFilterPushDown(gtFilter)//
.setRtAggrMetrics(gtRtAggrMetrics).setDynamicColumns(gtDynColumns)
- .setExprsPushDown(tupleExpressionList)//
+ .setExprsPushDown(tupleExpressionMap)//
.setAllowStorageAggregation(context.isNeedStorageAggregation())
.setAggCacheMemThreshold(cubeSegment.getConfig().getQueryCoprocessorMemGB())//
.setStoragePushDownLimit(context.getFinalPushDownLimit())
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.