You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/12/04 11:24:23 UTC
[2/5] kylin git commit: KYLIN-976 Extract fillTuple() methods
KYLIN-976 Extract fillTuple() methods
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/37f92de0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/37f92de0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/37f92de0
Branch: refs/heads/KYLIN-976
Commit: 37f92de0e4e7a737cd3cdc9c6c779e8628ded313
Parents: 63f376a
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Dec 3 14:10:33 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Dec 3 14:10:33 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/common/topn/TopNCounter.java | 4 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 2 +-
core-metadata/pom.xml | 6 +
.../org/apache/kylin/measure/MeasureType.java | 33 ++-
.../kylin/measure/basic/BasicMeasureType.java | 2 +-
.../kylin/measure/hllc/HLLCMeasureType.java | 2 +-
.../kylin/measure/topn/TopNMeasureType.java | 95 ++++++++-
.../kylin/metadata/model/FunctionDesc.java | 2 +-
.../org/apache/kylin/metadata/tuple/Tuple.java | 209 ++++++++++++++++++
.../apache/kylin/metadata/tuple/TupleInfo.java | 114 ++++++++++
core-storage/pom.xml | 7 -
.../org/apache/kylin/storage/IStorageQuery.java | 2 +-
.../storage/cache/CacheFledgedDynamicQuery.java | 2 +-
.../storage/cache/CacheFledgedStaticQuery.java | 2 +-
.../kylin/storage/cache/StreamSQLResult.java | 2 +-
.../storage/hybrid/HybridStorageQuery.java | 2 +-
.../org/apache/kylin/storage/tuple/Tuple.java | 210 -------------------
.../apache/kylin/storage/tuple/TupleInfo.java | 114 ----------
.../kylin/storage/cache/DynamicCacheTest.java | 4 +-
.../kylin/storage/cache/StaticCacheTest.java | 4 +-
.../kylin/storage/cache/StorageMockUtils.java | 2 +-
.../mr/steps/MergeCuboidFromStorageMapper.java | 2 +-
.../engine/mr/steps/MergeCuboidMapper.java | 2 +-
pom.xml | 2 +-
.../query/enumerator/LookupTableEnumerator.java | 2 +-
.../kylin/query/relnode/OLAPAggregateRel.java | 5 +-
.../apache/kylin/query/relnode/OLAPContext.java | 3 +-
server/pom.xml | 2 +-
.../org/apache/kylin/rest/StorageCacheTest.java | 2 +-
.../cube/v1/CubeSegmentTopNTupleIterator.java | 4 +-
.../hbase/cube/v1/CubeSegmentTupleIterator.java | 4 +-
.../storage/hbase/cube/v1/CubeStorageQuery.java | 2 +-
.../hbase/cube/v1/CubeTupleConverter.java | 4 +-
.../cube/v1/SerializedHBaseTupleIterator.java | 2 +-
.../storage/hbase/cube/v2/CubeStorageQuery.java | 34 +--
.../hbase/cube/v2/CubeTupleConverter.java | 83 ++++++--
.../v2/SequentialCubeTopNTupleIterator.java | 4 +-
.../cube/v2/SequentialCubeTupleIterator.java | 56 ++++-
.../hbase/ii/InvertedIndexStorageQuery.java | 2 +-
.../endpoint/EndpointTupleConverter.java | 4 +-
.../endpoint/EndpointTupleIterator.java | 4 +-
.../common/coprocessor/FilterBaseTest.java | 4 +-
.../common/coprocessor/FilterEvaluateTest.java | 2 +-
43 files changed, 612 insertions(+), 437 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java b/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
index 9536ef7..4a95c66 100644
--- a/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
+++ b/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
@@ -301,11 +301,11 @@ public class TopNCounter<T> implements Iterable<Counter<T>> {
public Iterator<Counter<T>> iterator() {
return new TopNCounterIterator();
}
-
+
/**
* Iterator from the tail (smallest) to head (biggest);
*/
- private class TopNCounterIterator implements Iterator {
+ private class TopNCounterIterator implements Iterator<Counter<T>> {
private ListNode2<Counter<T>> currentBNode;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 050aef2..25e7edd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -820,7 +820,7 @@ public class CubeDesc extends RootPersistentEntity {
for (MeasureDesc measure : measures) {
MeasureType aggrType = measure.getFunction().getMeasureType();
- result.addAll(aggrType.getColumnsNeedDictionary(measure));
+ result.addAll(aggrType.getColumnsNeedDictionary(measure.getFunction()));
}
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index c8836c4..3f1576a 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -41,6 +41,12 @@
<version>${project.parent.version}</version>
</dependency>
+ <dependency>
+ <groupId>net.sf.ehcache</groupId>
+ <artifactId>ehcache</artifactId>
+ <version>${ehcache.version}</version>
+ </dependency>
+
<!-- Env & Test -->
<dependency>
<groupId>org.apache.kylin</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index 0891d1e..d4b2700 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -19,9 +19,11 @@
package org.apache.kylin.measure;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.measure.basic.BasicMeasureFactory;
import org.apache.kylin.measure.hllc.HLLCAggregationFactory;
import org.apache.kylin.measure.topn.TopNMeasureFactory;
@@ -32,6 +34,8 @@ import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import com.google.common.collect.Maps;
@@ -82,8 +86,8 @@ abstract public class MeasureType {
abstract public MeasureAggregator<?> newAggregator();
- public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
- return null;
+ public List<TblColRef> getColumnsNeedDictionary(FunctionDesc functionDesc) {
+ return Collections.emptyList();
}
/* ============================================================================
@@ -102,10 +106,33 @@ abstract public class MeasureType {
abstract public boolean needRewrite();
- abstract public Class<?> getRewriteAggregationFunctionClass();
+ abstract public Class<?> getRewriteCalciteAggrFunctionClass();
/* ============================================================================
* Storage
* ---------------------------------------------------------------------------- */
+ public void beforeStorageQuery(MeasureDesc measureDesc, SQLDigest sqlDigest) {
+ }
+
+ public boolean needAdvancedTupleFilling() {
+ return false;
+ }
+
+ public void fillTupleSimply(Tuple tuple, int indexInTuple, Object measureValue) {
+ tuple.setMeasureValue(indexInTuple, measureValue);
+ }
+
+ public IAdvMeasureFiller getAdvancedTupleFiller(FunctionDesc function, TupleInfo returnTupleInfo, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ throw new UnsupportedOperationException();
+ }
+
+ public static interface IAdvMeasureFiller {
+
+ void reload(Object measureValue);
+
+ int getNumOfRows();
+
+ void fillTuplle(Tuple tuple, int row);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
index a6b36bb..fea6ba5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
@@ -122,7 +122,7 @@ public class BasicMeasureType extends MeasureType {
}
@Override
- public Class<?> getRewriteAggregationFunctionClass() {
+ public Class<?> getRewriteCalciteAggrFunctionClass() {
// TODO Auto-generated method stub
return null;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
index fd71c00..83fcf3b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
@@ -90,7 +90,7 @@ public class HLLCMeasureType extends MeasureType {
}
@Override
- public Class<?> getRewriteAggregationFunctionClass() {
+ public Class<?> getRewriteCalciteAggrFunctionClass() {
// TODO Auto-generated method stub
return null;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 77319be..01582d0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -20,6 +20,7 @@ package org.apache.kylin.measure.topn;
import java.util.Collection;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -39,9 +40,17 @@ import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
public class TopNMeasureType extends MeasureType {
+ private static final Logger logger = LoggerFactory.getLogger(TopNMeasureType.class);
+
private final DataType dataType;
public TopNMeasureType(DataType dataType) {
@@ -134,8 +143,8 @@ public class TopNMeasureType extends MeasureType {
}
@Override
- public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
- TblColRef literalCol = measureDesc.getFunction().getParameter().getColRefs().get(1);
+ public List<TblColRef> getColumnsNeedDictionary(FunctionDesc functionDesc) {
+ TblColRef literalCol = functionDesc.getParameter().getColRefs().get(1);
return Collections.singletonList(literalCol);
}
@@ -173,8 +182,88 @@ public class TopNMeasureType extends MeasureType {
}
@Override
- public Class<?> getRewriteAggregationFunctionClass() {
+ public Class<?> getRewriteCalciteAggrFunctionClass() {
return null;
}
+ @Override
+ public void beforeStorageQuery(MeasureDesc measureDesc, SQLDigest sqlDigest) {
+ FunctionDesc topnFunc = measureDesc.getFunction();
+ TblColRef topnLiteralCol = topnFunc.getTopNLiteralColumn();
+
+ if (sqlDigest.groupbyColumns.contains(topnLiteralCol) == false)
+ return;
+
+ if (sqlDigest.aggregations.size() != 1) {
+ throw new IllegalStateException("When query with topN, only one metrics is allowed.");
+ }
+
+ FunctionDesc origFunc = sqlDigest.aggregations.iterator().next();
+ if (origFunc.isSum() == false) {
+ throw new IllegalStateException("When query with topN, only SUM function is allowed.");
+ }
+
+ sqlDigest.aggregations = Lists.newArrayList(topnFunc);
+ sqlDigest.groupbyColumns.remove(topnLiteralCol);
+ sqlDigest.metricColumns.add(topnLiteralCol);
+ logger.info("Rewrite function " + origFunc + " to " + topnFunc);
+ }
+
+ @Override
+ public boolean needAdvancedTupleFilling() {
+ return true;
+ }
+
+ @Override
+ public void fillTupleSimply(Tuple tuple, int indexInTuple, Object measureValue) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IAdvMeasureFiller getAdvancedTupleFiller(FunctionDesc function, TupleInfo tupleInfo, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ final TblColRef literalCol = getTopNLiteralColumn(function);
+ final TblColRef numericCol = getTopNNumericColumn(function);
+ final Dictionary<String> topNColDict = dictionaryMap.get(literalCol);
+ final int literalTupleIdx = tupleInfo.hasColumn(literalCol) ? tupleInfo.getColumnIndex(literalCol) : -1;
+ // for TopN, the aggr must be SUM, so the number fill into the column position (without rewrite)
+ final int numericTupleIdx = tupleInfo.hasColumn(numericCol) ? tupleInfo.getColumnIndex(numericCol) : -1;
+
+ return new IAdvMeasureFiller() {
+ private TopNCounter<ByteArray> topNCounter;
+ private Iterator<Counter<ByteArray>> topNCounterIterator;
+ private int expectRow = 0;
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void reload(Object measureValue) {
+ this.topNCounter = (TopNCounter<ByteArray>) measureValue;
+ this.topNCounterIterator = topNCounter.iterator();
+ }
+
+ @Override
+ public int getNumOfRows() {
+ return topNCounter.size();
+ }
+
+ @Override
+ public void fillTuplle(Tuple tuple, int row) {
+ if (expectRow++ != row)
+ throw new IllegalStateException();
+
+ Counter<ByteArray> counter = topNCounterIterator.next();
+ int key = BytesUtil.readUnsigned(counter.getItem().array(), 0, counter.getItem().array().length);
+ String colValue = topNColDict.getValueFromId(key);
+ tuple.setDimensionValue(literalTupleIdx, colValue);
+ tuple.setMeasureValue(numericTupleIdx, counter.getCount());
+ }
+ };
+ }
+
+ private TblColRef getTopNNumericColumn(FunctionDesc functionDesc) {
+ return functionDesc.getParameter().getColRefs().get(0);
+ }
+
+ private TblColRef getTopNLiteralColumn(FunctionDesc functionDesc) {
+ return functionDesc.getParameter().getColRefs().get(1);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 17debda..104f3c7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -75,7 +75,7 @@ public class FunctionDesc {
parameter.setColRefs(colRefs);
// make sure sum/max/min returns the exact type as its input
- if (isSum() || isMax() || isMin() && (colRefs.size() > 0)) {
+ if ((isSum() || isMax() || isMin()) && colRefs.size() > 0) {
setReturnType(colRefs.get(0).getDatatype());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
new file mode 100644
index 0000000..c79e1f8
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.tuple;
+
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.List;
+
+import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf;
+
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.datatype.DoubleMutable;
+import org.apache.kylin.metadata.datatype.LongMutable;
+import org.apache.kylin.metadata.model.TblColRef;
+
+/**
+ * @author xjiang
+ */
+public class Tuple implements ITuple {
+
+ @IgnoreSizeOf
+ private final TupleInfo info;
+ private final Object[] values;
+
+ public Tuple(TupleInfo info) {
+ this.info = info;
+ this.values = new Object[info.size()];
+ }
+
+ public List<String> getAllFields() {
+ return info.getAllFields();
+ }
+
+ public List<TblColRef> getAllColumns() {
+ return info.getAllColumns();
+ }
+
+ public Object[] getAllValues() {
+ return values;
+ }
+
+ @Override
+ public ITuple makeCopy() {
+ Tuple ret = new Tuple(this.info);
+ for (int i = 0; i < this.values.length; ++i) {
+ if (this.values[i] == null) {
+ ret.values[i] = null;
+ } else if (this.values[i] instanceof HyperLogLogPlusCounter) {
+ ret.values[i] = new HyperLogLogPlusCounter((HyperLogLogPlusCounter) this.values[i]);
+ } else if (this.values[i] instanceof TopNCounter) {
+ ret.values[i] = null;
+ } else {
+ ret.values[i] = this.values[i];
+ }
+ }
+ return ret;
+ }
+
+ public TupleInfo getInfo() {
+ return info;
+ }
+
+ public String getFieldName(TblColRef col) {
+ return info.getFieldName(col);
+ }
+
+ public TblColRef getFieldColumn(String fieldName) {
+ return info.getColumn(fieldName);
+ }
+
+ public Object getValue(String fieldName) {
+ int index = info.getFieldIndex(fieldName);
+ return values[index];
+ }
+
+ public Object getValue(TblColRef col) {
+ int index = info.getColumnIndex(col);
+ return values[index];
+ }
+
+ public String getDataTypeName(int idx) {
+ return info.getDataTypeName(idx);
+ }
+
+ public void setDimensionValue(String fieldName, String fieldValue) {
+ setDimensionValue(info.getFieldIndex(fieldName), fieldValue);
+ }
+
+ public void setDimensionValue(int idx, String fieldValue) {
+ Object objectValue = convertOptiqCellValue(fieldValue, getDataTypeName(idx));
+ values[idx] = objectValue;
+ }
+
+ public void setMeasureValue(String fieldName, Object fieldValue) {
+ setMeasureValue(info.getFieldIndex(fieldName), fieldValue);
+ }
+
+ public void setMeasureValue(int idx, Object fieldValue) {
+ fieldValue = convertWritableToJava(fieldValue);
+
+ String dataType = getDataTypeName(idx);
+ // special handling for BigDecimal, allow double be aggregated as
+ // BigDecimal during cube build for best precision
+ if ("double".equals(dataType) && fieldValue instanceof BigDecimal) {
+ fieldValue = ((BigDecimal) fieldValue).doubleValue();
+ } else if ("integer".equals(dataType) && !(fieldValue instanceof Integer)) {
+ fieldValue = ((Number) fieldValue).intValue();
+ } else if ("float".equals(dataType) && fieldValue instanceof BigDecimal) {
+ fieldValue = ((BigDecimal) fieldValue).floatValue();
+ }
+ values[idx] = fieldValue;
+ }
+
+ private Object convertWritableToJava(Object o) {
+ if (o instanceof LongMutable)
+ o = ((LongMutable) o).get();
+ else if (o instanceof DoubleMutable)
+ o = ((DoubleMutable) o).get();
+ return o;
+ }
+
+ public boolean hasColumn(TblColRef column) {
+ return info.hasColumn(column);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ for (String field : info.getAllFields()) {
+ sb.append(field);
+ sb.append("=");
+ sb.append(getValue(field));
+ sb.append(",");
+ }
+ return sb.toString();
+ }
+
+ public static long epicDaysToMillis(int days) {
+ return 1L * days * (1000 * 3600 * 24);
+ }
+
+ public static int dateToEpicDays(String strValue) {
+ Date dateValue = DateFormat.stringToDate(strValue); // NOTE: forces GMT timezone
+ long millis = dateValue.getTime();
+ return (int) (millis / (1000 * 3600 * 24));
+ }
+
+ public static long getTs(ITuple row, TblColRef partitionCol) {
+ //ts column type differentiate
+ if (partitionCol.getDatatype().equals("date")) {
+ return Tuple.epicDaysToMillis(Integer.valueOf(row.getValue(partitionCol).toString()));
+ } else {
+ return Long.valueOf(row.getValue(partitionCol).toString());
+ }
+ }
+
+ public static Object convertOptiqCellValue(String strValue, String dataTypeName) {
+ if (strValue == null)
+ return null;
+
+ if ((strValue.equals("") || strValue.equals("\\N")) && !dataTypeName.equals("string"))
+ return null;
+
+ // TODO use data type enum instead of string comparison
+ if ("date".equals(dataTypeName)) {
+ // convert epoch time
+ return dateToEpicDays(strValue);// Optiq expects Integer instead of Long. by honma
+ } else if ("timestamp".equals(dataTypeName) || "datetime".equals(dataTypeName)) {
+ return Long.valueOf(DateFormat.stringToMillis(strValue));
+ } else if ("tinyint".equals(dataTypeName)) {
+ return Byte.valueOf(strValue);
+ } else if ("short".equals(dataTypeName) || "smallint".equals(dataTypeName)) {
+ return Short.valueOf(strValue);
+ } else if ("integer".equals(dataTypeName)) {
+ return Integer.valueOf(strValue);
+ } else if ("long".equals(dataTypeName) || "bigint".equals(dataTypeName)) {
+ return Long.valueOf(strValue);
+ } else if ("double".equals(dataTypeName)) {
+ return Double.valueOf(strValue);
+ } else if ("decimal".equals(dataTypeName)) {
+ return new BigDecimal(strValue);
+ } else if ("float".equals(dataTypeName)) {
+ return Float.valueOf(strValue);
+ } else if ("boolean".equals(dataTypeName)) {
+ return Boolean.valueOf(strValue);
+ } else {
+ return strValue;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
new file mode 100644
index 0000000..f7c3b57
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.tuple;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.metadata.model.TblColRef;
+
+/**
+ *
+ * @author xjiang
+ *
+ */
+public class TupleInfo {
+
+ private final Map<String, Integer> fieldMap;
+ private final Map<TblColRef, Integer> columnMap;
+
+ private final List<String> fields;
+ private final List<TblColRef> columns;
+ private final List<String> dataTypeNames;
+
+ public TupleInfo() {
+ fieldMap = new HashMap<String, Integer>();
+ columnMap = new HashMap<TblColRef, Integer>();
+ fields = new ArrayList<String>();
+ columns = new ArrayList<TblColRef>();
+ dataTypeNames = new ArrayList<String>();
+ }
+
+ public TblColRef getColumn(String fieldName) {
+ int idx = getFieldIndex(fieldName);
+ return columns.get(idx);
+ }
+
+ public int getColumnIndex(TblColRef col) {
+ return columnMap.get(col);
+ }
+
+ public String getDataTypeName(int index) {
+ return dataTypeNames.get(index);
+ }
+
+ public int getFieldIndex(String fieldName) {
+ return fieldMap.get(fieldName);
+ }
+
+ public boolean hasField(String fieldName) {
+ return fieldMap.containsKey(fieldName);
+ }
+
+ public String getFieldName(TblColRef col) {
+ int idx = columnMap.get(col);
+ return fields.get(idx);
+ }
+
+ public boolean hasColumn(TblColRef col) {
+ return columnMap.containsKey(col);
+ }
+
+ public void setField(String fieldName, TblColRef col, int index) {
+ fieldMap.put(fieldName, index);
+
+ if (col != null)
+ columnMap.put(col, index);
+
+ if (fields.size() > index)
+ fields.set(index, fieldName);
+ else
+ fields.add(index, fieldName);
+
+ if (columns.size() > index)
+ columns.set(index, col);
+ else
+ columns.add(index, col);
+
+ if (dataTypeNames.size() > index)
+ dataTypeNames.set(index, col.getType().getName());
+ else
+ dataTypeNames.add(index, col.getType().getName());
+ }
+
+ public List<String> getAllFields() {
+ return fields;
+ }
+
+ public List<TblColRef> getAllColumns() {
+ return columns;
+ }
+
+ public int size() {
+ return fields.size();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-storage/pom.xml
----------------------------------------------------------------------
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index e17d13f..ad440da 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -41,13 +41,6 @@
<version>${project.parent.version}</version>
</dependency>
- <dependency>
- <groupId>net.sf.ehcache</groupId>
- <artifactId>ehcache</artifactId>
- <version>2.8.1</version>
- </dependency>
-
-
<!-- Env & Test -->
<dependency>
<groupId>org.apache.kylin</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
index 9ad0af9..6b53b5b 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
@@ -20,7 +20,7 @@ package org.apache.kylin.storage;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.storage.tuple.TupleInfo;
+import org.apache.kylin.metadata.tuple.TupleInfo;
/**
*
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
index e1d5ee3..1481754 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
@@ -14,9 +14,9 @@ import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
import org.apache.kylin.metadata.tuple.TeeTupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
index af1b34c..57f33f6 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
@@ -10,9 +10,9 @@ import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
import org.apache.kylin.metadata.tuple.TeeTupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
index 1ad7229..74def9e 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
@@ -9,7 +9,7 @@ import javax.annotation.Nullable;
import org.apache.kylin.common.util.RangeUtil;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.storage.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.Tuple;
import com.google.common.base.Function;
import com.google.common.collect.Iterators;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java
index 8911c0e..0107434 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java
@@ -23,10 +23,10 @@ import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.CompoundTupleIterator;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.IStorageQuery;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.StorageFactory;
-import org.apache.kylin.storage.tuple.TupleInfo;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
deleted file mode 100644
index 3bca687..0000000
--- a/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.tuple;
-
-import java.math.BigDecimal;
-import java.util.Date;
-import java.util.List;
-
-import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf;
-
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.common.topn.TopNCounter;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
-import org.apache.kylin.metadata.datatype.LongMutable;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
-
-/**
- * @author xjiang
- */
-public class Tuple implements ITuple {
-
- @IgnoreSizeOf
- private final TupleInfo info;
- private final Object[] values;
-
- public Tuple(TupleInfo info) {
- this.info = info;
- this.values = new Object[info.size()];
- }
-
- public List<String> getAllFields() {
- return info.getAllFields();
- }
-
- public List<TblColRef> getAllColumns() {
- return info.getAllColumns();
- }
-
- public Object[] getAllValues() {
- return values;
- }
-
- @Override
- public ITuple makeCopy() {
- Tuple ret = new Tuple(this.info);
- for (int i = 0; i < this.values.length; ++i) {
- if (this.values[i] == null) {
- ret.values[i] = null;
- } else if (this.values[i] instanceof HyperLogLogPlusCounter) {
- ret.values[i] = new HyperLogLogPlusCounter((HyperLogLogPlusCounter) this.values[i]);
- } else if (this.values[i] instanceof TopNCounter) {
- ret.values[i] = null;
- } else {
- ret.values[i] = this.values[i];
- }
- }
- return ret;
- }
-
- public TupleInfo getInfo() {
- return info;
- }
-
- public String getFieldName(TblColRef col) {
- return info.getFieldName(col);
- }
-
- public TblColRef getFieldColumn(String fieldName) {
- return info.getColumn(fieldName);
- }
-
- public Object getValue(String fieldName) {
- int index = info.getFieldIndex(fieldName);
- return values[index];
- }
-
- public Object getValue(TblColRef col) {
- int index = info.getColumnIndex(col);
- return values[index];
- }
-
- public String getDataTypeName(int idx) {
- return info.getDataTypeName(idx);
- }
-
- public void setDimensionValue(String fieldName, String fieldValue) {
- setDimensionValue(info.getFieldIndex(fieldName), fieldValue);
- }
-
- public void setDimensionValue(int idx, String fieldValue) {
- Object objectValue = convertOptiqCellValue(fieldValue, getDataTypeName(idx));
- values[idx] = objectValue;
- }
-
- public void setMeasureValue(String fieldName, Object fieldValue) {
- setMeasureValue(info.getFieldIndex(fieldName), fieldValue);
- }
-
- public void setMeasureValue(int idx, Object fieldValue) {
- fieldValue = convertWritableToJava(fieldValue);
-
- String dataType = getDataTypeName(idx);
- // special handling for BigDecimal, allow double be aggregated as
- // BigDecimal during cube build for best precision
- if ("double".equals(dataType) && fieldValue instanceof BigDecimal) {
- fieldValue = ((BigDecimal) fieldValue).doubleValue();
- } else if ("integer".equals(dataType) && !(fieldValue instanceof Integer)) {
- fieldValue = ((Number) fieldValue).intValue();
- } else if ("float".equals(dataType) && fieldValue instanceof BigDecimal) {
- fieldValue = ((BigDecimal) fieldValue).floatValue();
- }
- values[idx] = fieldValue;
- }
-
- private Object convertWritableToJava(Object o) {
- if (o instanceof LongMutable)
- o = ((LongMutable) o).get();
- else if (o instanceof DoubleMutable)
- o = ((DoubleMutable) o).get();
- return o;
- }
-
- public boolean hasColumn(TblColRef column) {
- return info.hasColumn(column);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- for (String field : info.getAllFields()) {
- sb.append(field);
- sb.append("=");
- sb.append(getValue(field));
- sb.append(",");
- }
- return sb.toString();
- }
-
- public static long epicDaysToMillis(int days) {
- return 1L * days * (1000 * 3600 * 24);
- }
-
- public static int dateToEpicDays(String strValue) {
- Date dateValue = DateFormat.stringToDate(strValue); // NOTE: forces GMT timezone
- long millis = dateValue.getTime();
- return (int) (millis / (1000 * 3600 * 24));
- }
-
- public static long getTs(ITuple row, TblColRef partitionCol) {
- //ts column type differentiate
- if (partitionCol.getDatatype().equals("date")) {
- return Tuple.epicDaysToMillis(Integer.valueOf(row.getValue(partitionCol).toString()));
- } else {
- return Long.valueOf(row.getValue(partitionCol).toString());
- }
- }
-
- public static Object convertOptiqCellValue(String strValue, String dataTypeName) {
- if (strValue == null)
- return null;
-
- if ((strValue.equals("") || strValue.equals("\\N")) && !dataTypeName.equals("string"))
- return null;
-
- // TODO use data type enum instead of string comparison
- if ("date".equals(dataTypeName)) {
- // convert epoch time
- return dateToEpicDays(strValue);// Optiq expects Integer instead of Long. by honma
- } else if ("timestamp".equals(dataTypeName) || "datetime".equals(dataTypeName)) {
- return Long.valueOf(DateFormat.stringToMillis(strValue));
- } else if ("tinyint".equals(dataTypeName)) {
- return Byte.valueOf(strValue);
- } else if ("short".equals(dataTypeName) || "smallint".equals(dataTypeName)) {
- return Short.valueOf(strValue);
- } else if ("integer".equals(dataTypeName)) {
- return Integer.valueOf(strValue);
- } else if ("long".equals(dataTypeName) || "bigint".equals(dataTypeName)) {
- return Long.valueOf(strValue);
- } else if ("double".equals(dataTypeName)) {
- return Double.valueOf(strValue);
- } else if ("decimal".equals(dataTypeName)) {
- return new BigDecimal(strValue);
- } else if ("float".equals(dataTypeName)) {
- return Float.valueOf(strValue);
- } else if ("boolean".equals(dataTypeName)) {
- return Boolean.valueOf(strValue);
- } else {
- return strValue;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-storage/src/main/java/org/apache/kylin/storage/tuple/TupleInfo.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/tuple/TupleInfo.java b/core-storage/src/main/java/org/apache/kylin/storage/tuple/TupleInfo.java
deleted file mode 100644
index 6f50b6d..0000000
--- a/core-storage/src/main/java/org/apache/kylin/storage/tuple/TupleInfo.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.tuple;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- *
- * @author xjiang
- *
- */
-public class TupleInfo {
-
- private final Map<String, Integer> fieldMap;
- private final Map<TblColRef, Integer> columnMap;
-
- private final List<String> fields;
- private final List<TblColRef> columns;
- private final List<String> dataTypeNames;
-
- public TupleInfo() {
- fieldMap = new HashMap<String, Integer>();
- columnMap = new HashMap<TblColRef, Integer>();
- fields = new ArrayList<String>();
- columns = new ArrayList<TblColRef>();
- dataTypeNames = new ArrayList<String>();
- }
-
- public TblColRef getColumn(String fieldName) {
- int idx = getFieldIndex(fieldName);
- return columns.get(idx);
- }
-
- public int getColumnIndex(TblColRef col) {
- return columnMap.get(col);
- }
-
- public String getDataTypeName(int index) {
- return dataTypeNames.get(index);
- }
-
- public int getFieldIndex(String fieldName) {
- return fieldMap.get(fieldName);
- }
-
- public boolean hasField(String fieldName) {
- return fieldMap.containsKey(fieldName);
- }
-
- public String getFieldName(TblColRef col) {
- int idx = columnMap.get(col);
- return fields.get(idx);
- }
-
- public boolean hasColumn(TblColRef col) {
- return columnMap.containsKey(col);
- }
-
- public void setField(String fieldName, TblColRef col, int index) {
- fieldMap.put(fieldName, index);
-
- if (col != null)
- columnMap.put(col, index);
-
- if (fields.size() > index)
- fields.set(index, fieldName);
- else
- fields.add(index, fieldName);
-
- if (columns.size() > index)
- columns.set(index, col);
- else
- columns.add(index, col);
-
- if (dataTypeNames.size() > index)
- dataTypeNames.set(index, col.getType().getName());
- else
- dataTypeNames.add(index, col.getType().getName());
- }
-
- public List<String> getAllFields() {
- return fields;
- }
-
- public List<TblColRef> getAllColumns() {
- return columns;
- }
-
- public int size() {
- return fields.size();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
index 1011b1a..53e5f5b 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
@@ -16,10 +16,10 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
index 09e289b..b74432d 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
@@ -16,10 +16,10 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
index 2898f93..ab25ee2 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
@@ -13,7 +13,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.tuple.TupleInfo;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import com.google.common.collect.ImmutableList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
index bb0c073..070eb97 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
@@ -123,7 +123,7 @@ public class MergeCuboidFromStorageMapper extends KylinMapper<Object, Object, By
for (int i = 0; i < measureDescs.size(); i++) {
MeasureDesc measureDesc = measureDescs.get(i);
MeasureType measureType = measureDesc.getFunction().getMeasureType();
- if (measureType.getColumnsNeedDictionary(measureDesc).isEmpty() == false) {
+ if (measureType.getColumnsNeedDictionary(measureDesc.getFunction()).isEmpty() == false) {
dictMeasures.add(new Pair<Integer, MeasureIngester>(i, measureType.newIngester()));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index 401237b..b3f7f54 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -125,7 +125,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
for (int i = 0; i < measureDescs.size(); i++) {
MeasureDesc measureDesc = measureDescs.get(i);
MeasureType measureType = measureDesc.getFunction().getMeasureType();
- if (measureType.getColumnsNeedDictionary(measureDesc).isEmpty() == false) {
+ if (measureType.getColumnsNeedDictionary(measureDesc.getFunction()).isEmpty() == false) {
dictMeasures.add(new Pair<Integer, MeasureIngester>(i, measureType.newIngester()));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 31ab924..82dc06b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -75,7 +75,6 @@
<!-- Spark -->
<spark.version>1.3.0</spark.version>
-
<!-- Utility -->
<log4j.version>1.2.17</log4j.version>
<slf4j.version>1.6.4</slf4j.version>
@@ -89,6 +88,7 @@
<extendedset.version>1.3.4</extendedset.version>
<jetty.version>9.2.7.v20150116</jetty.version>
<kryo.version>3.0.3</kryo.version>
+ <ehcache.version>2.8.1</ehcache.version>
<!-- REST Service -->
<spring.framework.version>3.1.2.RELEASE</spring.framework.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
index 9e59f93..1c2bc2d 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
@@ -30,10 +30,10 @@ import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.tuple.Tuple;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.schema.OLAPTable;
import org.apache.kylin.storage.hybrid.HybridInstance;
-import org.apache.kylin.storage.tuple.Tuple;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 9aa70ca..cebfd12 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -73,7 +73,6 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
AGGR_FUNC_MAP.put("$SUM0", "SUM");
AGGR_FUNC_MAP.put("COUNT", "COUNT");
AGGR_FUNC_MAP.put("COUNT_DISTINCT", "COUNT_DISTINCT");
- AGGR_FUNC_MAP.put("HLL_COUNT", "COUNT_DISTINCT");
AGGR_FUNC_MAP.put("MAX", "MAX");
AGGR_FUNC_MAP.put("MIN", "MIN");
}
@@ -302,8 +301,8 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
SqlAggFunction newAgg = aggCall.getAggregation();
if (func.isCount()) {
newAgg = SqlStdOperatorTable.SUM0;
- } else if (func.getMeasureType().getRewriteAggregationFunctionClass() != null) {
- newAgg = createCustomAggFunction(fieldType, func.getMeasureType().getRewriteAggregationFunctionClass());
+ } else if (func.getMeasureType().getRewriteCalciteAggrFunctionClass() != null) {
+ newAgg = createCustomAggFunction(fieldType, func.getMeasureType().getRewriteCalciteAggrFunctionClass());
}
// rebuild aggregate call
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 6865457..6de1790 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import com.google.common.collect.Lists;
+
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -36,9 +37,9 @@ import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.query.schema.OLAPSchema;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.TupleInfo;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index f2f9e32..304bb73 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -206,7 +206,7 @@
<dependency>
<groupId>net.sf.ehcache</groupId>
<artifactId>ehcache</artifactId>
- <version>2.8.1</version>
+ <version>${ehcache.version}</version>
</dependency>
<dependency>
<groupId>cglib</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java b/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java
index e781e00..b6cc6d0 100644
--- a/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java
@@ -24,12 +24,12 @@ import net.sf.ehcache.Element;
import org.apache.commons.lang.NotImplementedException;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.rest.service.ServiceTestBase;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery;
import org.apache.kylin.storage.cache.CacheFledgedStaticQuery;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTopNTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTopNTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTopNTupleIterator.java
index a8b1d02..16748a6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTopNTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTopNTupleIterator.java
@@ -23,11 +23,11 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
import org.apache.kylin.storage.translate.HBaseKeyRange;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import java.util.Iterator;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
index 22f7017..4acd8f8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
@@ -42,13 +42,13 @@ import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.apache.kylin.storage.hbase.cube.v1.filter.FuzzyRowFilterV2;
import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
import org.apache.kylin.storage.translate.HBaseKeyRange;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 0cd9014..13c75de 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -59,6 +59,7 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -67,7 +68,6 @@ import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
import org.apache.kylin.storage.translate.ColumnValueRange;
import org.apache.kylin.storage.translate.DerivedFilterTranslator;
import org.apache.kylin.storage.translate.HBaseKeyRange;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
index 85f31c4..6be89d6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
@@ -23,9 +23,9 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
index 0983689..3f92cb0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
@@ -33,11 +33,11 @@ import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.exception.ScanOutOfLimitException;
import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
import org.apache.kylin.storage.translate.HBaseKeyRange;
-import org.apache.kylin.storage.tuple.TupleInfo;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index 0c8c3bd..3dbbce1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -15,6 +15,7 @@ import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
+import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
@@ -26,10 +27,10 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.translate.DerivedFilterTranslator;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,7 +55,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
// check whether this is a TopN query
- checkAndRewriteTopN(sqlDigest);
+ notifyBeforeStorageQuery(sqlDigest);
Collection<TblColRef> groups = sqlDigest.groupbyColumns;
TupleFilter filter = sqlDigest.filter;
@@ -393,33 +394,10 @@ public class CubeStorageQuery implements ICachableStorageQuery {
return false;
}
- private void checkAndRewriteTopN(SQLDigest sqlDigest) {
- FunctionDesc topnFunc = null;
- TblColRef topnLiteralCol = null;
+ private void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
for (MeasureDesc measure : cubeDesc.getMeasures()) {
- FunctionDesc func = measure.getFunction();
- if (func.isTopN() && sqlDigest.groupbyColumns.contains(func.getTopNLiteralColumn())) {
- topnFunc = func;
- topnLiteralCol = func.getTopNLiteralColumn();
- }
- }
-
- // if TopN is not involved
- if (topnFunc == null)
- return;
-
- if (sqlDigest.aggregations.size() != 1) {
- throw new IllegalStateException("When query with topN, only one metrics is allowed.");
- }
-
- FunctionDesc origFunc = sqlDigest.aggregations.iterator().next();
- if (origFunc.isSum() == false) {
- throw new IllegalStateException("When query with topN, only SUM function is allowed.");
+ MeasureType measureType = measure.getFunction().getMeasureType();
+ measureType.beforeStorageQuery(measure, sqlDigest);
}
-
- sqlDigest.aggregations = Lists.newArrayList(topnFunc);
- sqlDigest.groupbyColumns.remove(topnLiteralCol);
- sqlDigest.metricColumns.add(topnLiteralCol);
- logger.info("Rewrite function " + origFunc + " to " + topnFunc);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
index 8fa63d3..a9e5a65 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
@@ -24,7 +24,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import org.apache.hadoop.hbase.client.Result;
import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.Array;
@@ -38,12 +37,15 @@ import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.MeasureType.IAdvMeasureFiller;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
* convert GTRecord to tuple
@@ -57,7 +59,12 @@ public class CubeTupleConverter {
final int[] gtColIdx;
final int[] tupleIdx;
- final Object[] tmpValues;
+ final Object[] gtValues;
+ final MeasureType[] measureTypes;
+
+ final List<IAdvMeasureFiller> advMeasureFillers;
+ final List<Integer> advMeasureIndexInGTValues;
+
final int nSelectedDims;
final TblColRef topNCol;
int topNColTupleIdx;
@@ -78,8 +85,14 @@ public class CubeTupleConverter {
nSelectedDims = selectedDimensions.size();
gtColIdx = new int[selectedDimensions.size() + selectedMetrics.size()];
tupleIdx = new int[selectedDimensions.size() + selectedMetrics.size()];
- tmpValues = new Object[selectedDimensions.size() + selectedMetrics.size()];
-
+ gtValues = new Object[selectedDimensions.size() + selectedMetrics.size()];
+
+ // measure types don't have this many, but aligned length make programming easier
+ measureTypes = new MeasureType[selectedDimensions.size() + selectedMetrics.size()];
+
+ advMeasureFillers = Lists.newArrayListWithCapacity(1);
+ advMeasureIndexInGTValues = Lists.newArrayListWithCapacity(1);
+
int iii = 0;
// pre-calculate dimension index mapping to tuple
@@ -93,6 +106,7 @@ public class CubeTupleConverter {
for (FunctionDesc metric : selectedMetrics) {
int i = mapping.getIndexOf(metric);
gtColIdx[iii] = i;
+
if (metric.needRewrite()) {
String rewriteFieldName = metric.getRewriteFieldName();
tupleIdx[iii] = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
@@ -102,6 +116,16 @@ public class CubeTupleConverter {
TblColRef col = metric.getParameter().getColRefs().get(0);
tupleIdx[iii] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
}
+
+ MeasureType measureType = metric.getMeasureType();
+ if (measureType.needAdvancedTupleFilling()) {
+ Map<TblColRef, Dictionary<String>> dictionaryMap = buildDictionaryMap(measureType.getColumnsNeedDictionary(metric));
+ advMeasureFillers.add(measureType.getAdvancedTupleFiller(metric, returnTupleInfo, dictionaryMap));
+ advMeasureIndexInGTValues.add(iii);
+ } else {
+ measureTypes[iii] = measureType;
+ }
+
iii++;
}
@@ -126,29 +150,49 @@ public class CubeTupleConverter {
}
}
- public void translateResult(GTRecord record, Tuple tuple) {
+ // load only needed dictionaries
+ private Map<TblColRef, Dictionary<String>> buildDictionaryMap(List<TblColRef> columnsNeedDictionary) {
+ Map<TblColRef, Dictionary<String>> result = Maps.newHashMap();
+ for (TblColRef col : columnsNeedDictionary) {
+ result.put(col, cubeSeg.getDictionary(col));
+ }
+ return result;
+ }
+
+ public List<IAdvMeasureFiller> translateResult(GTRecord record, Tuple tuple) {
- record.getValues(gtColIdx, tmpValues);
+ record.getValues(gtColIdx, gtValues);
// dimensions
for (int i = 0; i < nSelectedDims; i++) {
int ti = tupleIdx[i];
if (ti >= 0) {
- tuple.setDimensionValue(ti, toString(tmpValues[i]));
+ tuple.setDimensionValue(ti, toString(gtValues[i]));
}
}
// measures
for (int i = nSelectedDims; i < gtColIdx.length; i++) {
int ti = tupleIdx[i];
- if (ti >= 0) {
- tuple.setMeasureValue(ti, tmpValues[i]);
+ if (ti >= 0 && measureTypes[i] != null) {
+ measureTypes[i].fillTupleSimply(tuple, ti, gtValues[i]);
}
}
// derived
for (IDerivedColumnFiller filler : derivedColFillers) {
- filler.fillDerivedColumns(tmpValues, tuple);
+ filler.fillDerivedColumns(gtValues, tuple);
+ }
+
+ // advanced measure filling, due to possible row split, will complete at caller side
+ if (advMeasureFillers.isEmpty()) {
+ return null;
+ } else {
+ for (int i = 0; i < advMeasureFillers.size(); i++) {
+ Object measureValue = gtValues[advMeasureIndexInGTValues.get(i)];
+ advMeasureFillers.get(i).reload(measureValue);
+ }
+ return advMeasureFillers;
}
}
@@ -158,15 +202,16 @@ public class CubeTupleConverter {
assert (topNCounterObj instanceof TopNCounter);
return new TopNCounterTupleIterator(tuple, (TopNCounter) topNCounterObj);
}
+
private interface IDerivedColumnFiller {
- public void fillDerivedColumns(Object[] tmpValues, Tuple tuple);
+ public void fillDerivedColumns(Object[] gtValues, Tuple tuple);
}
private IDerivedColumnFiller newDerivedColumnFiller(TblColRef[] hostCols, final DeriveInfo deriveInfo) {
boolean allHostsPresent = true;
final int[] hostTmpIdx = new int[hostCols.length];
for (int i = 0; i < hostCols.length; i++) {
- hostTmpIdx[i] = indexOnTheTmpValues(hostCols[i]);
+ hostTmpIdx[i] = indexOnTheGTValues(hostCols[i]);
allHostsPresent = allHostsPresent && hostTmpIdx[i] >= 0;
}
@@ -198,9 +243,9 @@ public class CubeTupleConverter {
}
@Override
- public void fillDerivedColumns(Object[] tmpValues, Tuple tuple) {
+ public void fillDerivedColumns(Object[] gtValues, Tuple tuple) {
for (int i = 0; i < hostTmpIdx.length; i++) {
- lookupKey.data[i] = CubeTupleConverter.toString(tmpValues[hostTmpIdx[i]]);
+ lookupKey.data[i] = CubeTupleConverter.toString(gtValues[hostTmpIdx[i]]);
}
String[] lookupRow = lookupTable.getRow(lookupKey);
@@ -224,9 +269,9 @@ public class CubeTupleConverter {
case PK_FK:
return new IDerivedColumnFiller() {
@Override
- public void fillDerivedColumns(Object[] tmpValues, Tuple tuple) {
+ public void fillDerivedColumns(Object[] gtValues, Tuple tuple) {
// composite keys are split, so only copy [0] is enough, see CubeDesc.initDimensionColumns()
- tuple.setDimensionValue(derivedTupleIdx[0], CubeTupleConverter.toString(tmpValues[hostTmpIdx[0]]));
+ tuple.setDimensionValue(derivedTupleIdx[0], CubeTupleConverter.toString(gtValues[hostTmpIdx[0]]));
}
};
default:
@@ -234,7 +279,7 @@ public class CubeTupleConverter {
}
}
- private int indexOnTheTmpValues(TblColRef col) {
+ private int indexOnTheGTValues(TblColRef col) {
List<TblColRef> cuboidDims = cuboid.getColumns();
int cuboidIdx = cuboidDims.indexOf(col);
for (int i = 0; i < gtColIdx.length; i++) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
index f162be6..9eeae4e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
@@ -3,9 +3,9 @@ package org.apache.kylin.storage.hbase.cube.v2;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import java.util.Iterator;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
index ddaafd5..5bf5e95 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
@@ -8,13 +8,14 @@ import java.util.Set;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.measure.MeasureType.IAdvMeasureFiller;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,6 +35,10 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
protected Iterator<GTRecord> curRecordIterator;
protected CubeTupleConverter curTupleConverter;
protected Tuple next;
+
+ private List<IAdvMeasureFiller> advMeasureFillers;
+ private int advMeasureRowsRemaining;
+ private int advMeasureRowIndex;
private int scanCount;
private int scanCountDelta;
@@ -53,7 +58,19 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
public boolean hasNext() {
if (next != null)
return true;
+
+ // consume any left rows from advanced measure filler
+ if (advMeasureRowsRemaining > 0) {
+ for (IAdvMeasureFiller filler : advMeasureFillers) {
+ filler.fillTuplle(tuple, advMeasureRowIndex);
+ }
+ advMeasureRowIndex++;
+ advMeasureRowsRemaining--;
+ next = tuple;
+ return true;
+ }
+ // get the next GTRecord
if (curScanner == null) {
if (scannerIterator.hasNext()) {
curScanner = scannerIterator.next();
@@ -63,18 +80,39 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
return false;
}
}
-
- if (curRecordIterator.hasNext()) {
- curTupleConverter.translateResult(curRecordIterator.next(), tuple);
- next = tuple;
- return true;
- } else {
+ if (curRecordIterator.hasNext() == false) {
close(curScanner);
curScanner = null;
curRecordIterator = null;
curTupleConverter = null;
return hasNext();
}
+
+ // now we have a GTRecord
+ GTRecord curRecord = curRecordIterator.next();
+
+ // translate into tuple
+ List<IAdvMeasureFiller> advMeasureFillers = curTupleConverter.translateResult(curRecord, tuple);
+
+ // the simple case
+ if (advMeasureFillers == null) {
+ next = tuple;
+ return true;
+ }
+
+ // advanced measure filling, like TopN, will produce multiple tuples out of one record
+ advMeasureRowsRemaining = -1;
+ for (IAdvMeasureFiller filler : advMeasureFillers) {
+ if (advMeasureRowsRemaining < 0)
+ advMeasureRowsRemaining = filler.getNumOfRows();
+ if (advMeasureRowsRemaining != filler.getNumOfRows())
+ throw new IllegalStateException();
+ }
+ if (advMeasureRowsRemaining < 0)
+ throw new IllegalStateException();
+
+ advMeasureRowIndex = 0;
+ return hasNext();
}
@Override
@@ -119,7 +157,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
logger.error("Exception when close CubeScanner", e);
}
}
-
+
public int getScanCount() {
return scanCount;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
index eef7143..5afb62d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
@@ -25,11 +25,11 @@ import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.EndpointTupleIterator;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java
index 9640af6..1073021 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java
@@ -6,8 +6,8 @@ import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
public class EndpointTupleConverter {
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
index 2fd0b4f..e638386 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
@@ -48,6 +48,8 @@ import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.cache.TsConditionExtractor;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
@@ -55,8 +57,6 @@ import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
import org.apache.kylin.storage.hbase.common.coprocessor.FilterDecorator;
import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.generated.IIProtos;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/37f92de0/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
index 483b8be..92e9699 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
@@ -36,8 +36,8 @@ import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
/**
* @author xjiang