You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/06/06 10:30:36 UTC
[2/3] kylin git commit: KYLIN-1478 support non-dic encoding
KYLIN-1478 support non-dic encoding
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9113a467
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9113a467
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9113a467
Branch: refs/heads/master
Commit: 9113a4675430a0baffb8d133ba29465c5cd70b4a
Parents: a490224
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jun 6 17:37:11 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jun 6 18:28:56 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/cube/model/RowKeyColDesc.java | 11 +-
.../metadata/measure/TopNMeasureTypeTest.java | 62 ++++++
.../kylin/dimension/DimensionEncoding.java | 13 ++
.../apache/kylin/dimension/IntegerDimEnc.java | 16 ++
.../kylin/measure/topn/TopNMeasureType.java | 202 +++++++++++++------
.../kylin/metadata/model/FunctionDesc.java | 14 +-
.../kylin/metadata/model/MeasureDesc.java | 2 +
...t_kylin_cube_without_slr_left_join_desc.json | 3 +-
.../test_kylin_inner_join_model_desc.json | 17 +-
.../test_kylin_left_join_model_desc.json | 16 +-
.../test_case_data/sandbox/kylin_hive_conf.xml | 12 +-
11 files changed, 261 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 7e03355..e1be041 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.cube.model;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DimensionEncoding;
import org.apache.kylin.dimension.DimensionEncodingFactory;
import org.apache.kylin.metadata.model.TblColRef;
@@ -57,13 +58,9 @@ public class RowKeyColDesc {
public void init() {
Preconditions.checkState(StringUtils.isNotEmpty(this.encoding));
- String[] parts = this.encoding.split("\\s*[(),:]\\s*");
- if (parts == null || parts.length == 0 || parts[0].isEmpty())
- throw new IllegalArgumentException("Not supported row key col encoding: '" + this.encoding + "'");
-
- this.encodingName = parts[0];
- this.encodingArgs = parts[parts.length - 1].isEmpty() //
- ? StringUtil.subArray(parts, 1, parts.length - 1) : StringUtil.subArray(parts, 1, parts.length);
+ Object[] encodingConf = DimensionEncoding.parseEncodingConf(this.encoding);
+ this.encodingName = (String) encodingConf[0];
+ this.encodingArgs = (String[])encodingConf[1];
if (!DimensionEncodingFactory.isVaildEncoding(this.encodingName))
throw new IllegalArgumentException("Not supported row key col encoding: '" + this.encoding + "'");
http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
new file mode 100644
index 0000000..3d3e7d0
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
@@ -0,0 +1,62 @@
+package org.apache.kylin.metadata.measure;
+
+import static org.junit.Assert.*;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.measure.MeasureTypeFactory;
+import org.apache.kylin.measure.topn.TopNMeasureType;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+/**
+ * Created by shishaofeng on 6/6/16.
+ */
+public class TopNMeasureTypeTest extends LocalFileMetadataTestCase {
+
+
+ @Before
+ public void setup() {
+ this.createTestMetadata();
+
+ }
+
+ @After
+ public void clear() {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void test() {
+
+ CubeDesc desc = CubeDescManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
+
+ MeasureDesc topSellerMeasure = null;
+
+ for (MeasureDesc measureDesc : desc.getMeasures()) {
+ if (measureDesc.getName().equals("TOP_SELLER")) {
+ topSellerMeasure = measureDesc;
+ break;
+ }
+ }
+ TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create(topSellerMeasure.getFunction().getExpression(), topSellerMeasure.getFunction().getReturnDataType());
+
+
+ topSellerMeasure.getFunction().getConfiguration().clear();
+ List<TblColRef> colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction());
+
+ assertTrue(colsNeedDict != null && colsNeedDict.size() == 1);
+
+ TblColRef sellerColRef = topSellerMeasure.getFunction().getParameter().getColRefs().get(1);
+ topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getName(), "int:6");
+ colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction());
+
+ assertTrue(colsNeedDict.size() == 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
index 5c32114..87dfc08 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
@@ -20,6 +20,7 @@ package org.apache.kylin.dimension;
import java.io.Externalizable;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
/**
@@ -49,6 +50,18 @@ public abstract class DimensionEncoding implements Externalizable {
return true;
}
+ public static Object[] parseEncodingConf(String encoding) {
+ String[] parts = encoding.split("\\s*[(),:]\\s*");
+ if (parts == null || parts.length == 0 || parts[0].isEmpty())
+ throw new IllegalArgumentException("Not supported row key col encoding: '" + encoding + "'");
+
+ final String encodingName = parts[0];
+ final String[] encodingArgs = parts[parts.length - 1].isEmpty() //
+ ? StringUtil.subArray(parts, 1, parts.length - 1) : StringUtil.subArray(parts, 1, parts.length);
+
+ return new Object[] {encodingName, encodingArgs};
+ }
+
/** return the fixed length of encoded bytes */
abstract public int getLengthOfEncoding();
http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index 9701447..3f01948 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -173,4 +173,20 @@ public class IntegerDimEnc extends DimensionEncoding {
fixedLen = in.readShort();
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ IntegerDimEnc that = (IntegerDimEnc) o;
+
+ return fixedLen == that.fixedLen;
+
+ }
+
+ @Override
+ public int hashCode() {
+ return fixedLen;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 009edef..0ff5b3a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -18,24 +18,21 @@
package org.apache.kylin.measure.topn;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
+import java.util.*;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.util.*;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DimensionEncoding;
+import org.apache.kylin.dimension.DimensionEncodingFactory;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.model.*;
import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.Tuple;
@@ -111,21 +108,38 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
@Override
public MeasureIngester<TopNCounter<ByteArray>> newIngester() {
return new MeasureIngester<TopNCounter<ByteArray>>() {
+
+ private DimensionEncoding[] dimensionEncodings = null;
+ private List<TblColRef> literalCols = null;
+ private int keyLength = 0;
+
+ private DimensionEncoding[] newDimensionEncodings = null;
+ private int newKeyLength = 0;
+ private boolean needReEncode = true;
+
@Override
public TopNCounter<ByteArray> valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
- if (values.length != 2)
- throw new IllegalArgumentException();
-
double counter = values[0] == null ? 0 : Double.parseDouble(values[0]);
- String literal = values[1];
- // encode literal using dictionary
- TblColRef literalCol = getTopNLiteralColumn(measureDesc.getFunction());
- Dictionary<String> dictionary = dictionaryMap.get(literalCol);
- int keyEncodedValue = dictionary.getIdFromValue(literal);
+ if (dimensionEncodings == null) {
+ literalCols = getTopNLiteralColumn(measureDesc.getFunction());
+ dimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, dictionaryMap);
+ for (DimensionEncoding encoding : dimensionEncodings) {
+ keyLength += encoding.getLengthOfEncoding();
+ }
- ByteArray key = new ByteArray(dictionary.getSizeOfId());
- BytesUtil.writeUnsigned(keyEncodedValue, key.array(), 0, dictionary.getSizeOfId());
+ if (values.length != (literalCols.size() + 1)) {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ final ByteArray key = new ByteArray(keyLength);
+ int offset = 0;
+ for (int i=0; i<dimensionEncodings.length; i++) {
+ byte[] valueBytes = Bytes.toBytes(values[i+1]);
+ dimensionEncodings[i].encode(valueBytes, valueBytes.length, key.array(), offset);
+ offset += dimensionEncodings[i].getLengthOfEncoding();
+ }
TopNCounter<ByteArray> topNCounter = new TopNCounter<ByteArray>(dataType.getPrecision() * TopNCounter.EXTRA_SPACE_RATE);
topNCounter.offer(key, counter);
@@ -136,30 +150,51 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
public TopNCounter<ByteArray> reEncodeDictionary(TopNCounter<ByteArray> value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
TopNCounter<ByteArray> topNCounter = value;
- TblColRef colRef = getTopNLiteralColumn(measureDesc.getFunction());
- Dictionary<String> sourceDict = oldDicts.get(colRef);
- Dictionary<String> mergedDict = newDicts.get(colRef);
+ if (newDimensionEncodings == null) {
+ literalCols = getTopNLiteralColumn(measureDesc.getFunction());
+ dimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, oldDicts);
+ keyLength = 0;
+ boolean hasDictEncoding = false;
+ for (DimensionEncoding encoding : dimensionEncodings) {
+ keyLength += encoding.getLengthOfEncoding();
+ if (encoding instanceof DictionaryDimEnc) {
+ hasDictEncoding = true;
+ }
+ }
+
+ newDimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, newDicts);
+ newKeyLength = 0;
+ for (DimensionEncoding encoding : newDimensionEncodings) {
+ newKeyLength += encoding.getLengthOfEncoding();
+ }
+
+ needReEncode = hasDictEncoding;
+ }
+
+ if (needReEncode == false) {
+ // no need re-encode
+ return topNCounter;
+ }
int topNSize = topNCounter.size();
- byte[] newIdBuf = new byte[topNSize * mergedDict.getSizeOfId()];
- byte[] literal = new byte[sourceDict.getSizeOfValue()];
+ byte[] newIdBuf = new byte[topNSize * newKeyLength];
int bufOffset = 0;
for (Counter<ByteArray> c : topNCounter) {
- int oldId = BytesUtil.readUnsigned(c.getItem().array(), c.getItem().offset(), c.getItem().length());
- int newId;
- int size = sourceDict.getValueBytesFromId(oldId, literal, 0);
- if (size < 0) {
- newId = mergedDict.nullId();
- } else {
- newId = mergedDict.getIdFromValueBytes(literal, 0, size);
+ int offset = c.getItem().offset();
+ int innerBuffOffset = 0;
+ for (int i = 0; i < dimensionEncodings.length; i++) {
+ String dimValue = dimensionEncodings[i].decode(c.getItem().array(), offset, dimensionEncodings[i].getLengthOfEncoding());
+ byte[] dimValueBytes = Bytes.toBytes(dimValue);
+ newDimensionEncodings[i].encode(dimValueBytes, dimValueBytes.length, newIdBuf, bufOffset + innerBuffOffset);
+ innerBuffOffset += newDimensionEncodings[i].getLengthOfEncoding();
+ offset += dimensionEncodings[i].getLengthOfEncoding();
}
- BytesUtil.writeUnsigned(newId, newIdBuf, bufOffset, mergedDict.getSizeOfId());
- c.getItem().set(newIdBuf, bufOffset, mergedDict.getSizeOfId());
- bufOffset += mergedDict.getSizeOfId();
+ c.getItem().set(newIdBuf, bufOffset, newKeyLength);
+ bufOffset += newKeyLength;
}
- return value;
+ return topNCounter;
}
};
}
@@ -171,8 +206,18 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
@Override
public List<TblColRef> getColumnsNeedDictionary(FunctionDesc functionDesc) {
- TblColRef literalCol = functionDesc.getParameter().getColRefs().get(1);
- return Collections.singletonList(literalCol);
+ List <TblColRef> columnsNeedDict = Lists.newArrayList();
+ List <TblColRef> allCols = functionDesc.getParameter().getColRefs();
+ int start = (functionDesc.getParameter().isColumnType() == true) ? 1 : 0;
+ for (int i = start; i < allCols.size(); i++) {
+ TblColRef tblColRef = allCols.get(i);
+ String encoding = functionDesc.getConfiguration().get(CONFIG_ENCODING_PREFIX + tblColRef.getName());
+ if(StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) {
+ columnsNeedDict.add(tblColRef);
+ }
+ }
+
+ return columnsNeedDict;
}
@Override
@@ -189,13 +234,13 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
if (isTopNCompatibleSum(topN.getFunction(), onlyFunction) == false)
return null;
- TblColRef literalCol = getTopNLiteralColumn(topN.getFunction());
- if (unmatchedDimensions.contains(literalCol) == false)
+ List<TblColRef> literalCol = getTopNLiteralColumn(topN.getFunction());
+ if (unmatchedDimensions.containsAll(literalCol) == false)
return null;
- if (digest.groupbyColumns.contains(literalCol) == false)
+ if (digest.groupbyColumns.containsAll(literalCol) == false)
return null;
- unmatchedDimensions.remove(literalCol);
+ unmatchedDimensions.removeAll(literalCol);
unmatchedAggregations.remove(onlyFunction);
return new CapabilityInfluence() {
@Override
@@ -212,11 +257,16 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
if (!isTopN(topN) || !sum.isSum())
return false;
- if (sum.getParameter().getColRefs().isEmpty())
- return false;
+ TblColRef topnNumCol = getTopNNumericColumn(topN);
+
+ if (sum.getParameter().getColRefs().isEmpty()) {
+ if (topnNumCol == null && sum.getParameter().getValue().equals(topN.getParameter().getValue()))
+ return true;
+ else
+ return false;
+ }
TblColRef sumCol = sum.getParameter().getColRefs().get(0);
- TblColRef topnNumCol = getTopNNumericColumn(topN);
return sumCol.equals(topnNumCol);
}
@@ -233,9 +283,9 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
@Override
public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) {
FunctionDesc topnFunc = measureDesc.getFunction();
- TblColRef topnLiteralCol = getTopNLiteralColumn(topnFunc);
+ List<TblColRef> topnLiteralCol = getTopNLiteralColumn(topnFunc);
- if (sqlDigest.groupbyColumns.contains(topnLiteralCol) == false)
+ if (sqlDigest.groupbyColumns.containsAll(topnLiteralCol) == false)
return;
if (sqlDigest.aggregations.size() > 1) {
@@ -251,8 +301,8 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
}
sqlDigest.aggregations = Lists.newArrayList(topnFunc);
- sqlDigest.groupbyColumns.remove(topnLiteralCol);
- sqlDigest.metricColumns.add(topnLiteralCol);
+ sqlDigest.groupbyColumns.removeAll(topnLiteralCol);
+ sqlDigest.metricColumns.addAll(topnLiteralCol);
}
@Override
@@ -267,12 +317,17 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
@Override
public IAdvMeasureFiller getAdvancedTupleFiller(FunctionDesc function, TupleInfo tupleInfo, Map<TblColRef, Dictionary<String>> dictionaryMap) {
- final TblColRef literalCol = getTopNLiteralColumn(function);
+ final List<TblColRef> literalCols = getTopNLiteralColumn(function);
final TblColRef numericCol = getTopNNumericColumn(function);
- final Dictionary<String> topNColDict = dictionaryMap.get(literalCol);
- final int literalTupleIdx = tupleInfo.hasColumn(literalCol) ? tupleInfo.getColumnIndex(literalCol) : -1;
+ final int[] literalTupleIdx = new int[literalCols.size()];
+ final DimensionEncoding[] dimensionEncodings = getDimensionEncodings(function, literalCols, dictionaryMap);
+ for (int i=0; i< literalCols.size(); i++) {
+ TblColRef colRef = literalCols.get(i);
+ literalTupleIdx[i] = tupleInfo.hasColumn(colRef) ? tupleInfo.getColumnIndex(colRef) : -1;
+ }
+
// for TopN, the aggr must be SUM, so the number fill into the column position (without rewrite)
- final int numericTupleIdx = tupleInfo.hasColumn(numericCol) ? tupleInfo.getColumnIndex(numericCol) : -1;
+ final int numericTupleIdx = (numericCol != null && tupleInfo.hasColumn(numericCol)) ? tupleInfo.getColumnIndex(numericCol) : -1;
return new IAdvMeasureFiller() {
private TopNCounter<ByteArray> topNCounter;
private Iterator<Counter<ByteArray>> topNCounterIterator;
@@ -297,23 +352,50 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
throw new IllegalStateException();
Counter<ByteArray> counter = topNCounterIterator.next();
- int key = BytesUtil.readUnsigned(counter.getItem().array(), counter.getItem().offset(), counter.getItem().length());
- String colValue = topNColDict.getValueFromId(key);
- tuple.setDimensionValue(literalTupleIdx, colValue);
+ int offset = counter.getItem().offset();
+ for (int i = 0; i< dimensionEncodings.length; i++) {
+ String colValue = dimensionEncodings[i].decode(counter.getItem().array(), offset, dimensionEncodings[i].getLengthOfEncoding());
+ tuple.setDimensionValue(literalTupleIdx[i], colValue);
+ offset += dimensionEncodings[i].getLengthOfEncoding();
+ }
tuple.setMeasureValue(numericTupleIdx, counter.getCount());
}
};
}
+ private static DimensionEncoding[] getDimensionEncodings(FunctionDesc function, List<TblColRef> literalCols, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ final DimensionEncoding[] dimensionEncodings = new DimensionEncoding[literalCols.size()];
+ for (int i=0; i< literalCols.size(); i++) {
+ TblColRef colRef = literalCols.get(i);
+ String encoding = function.getConfiguration().get(TopNMeasureType.CONFIG_ENCODING_PREFIX + colRef.getName());
+ if(StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) {
+ dimensionEncodings[i] = new DictionaryDimEnc(dictionaryMap.get(colRef));
+ } else {
+ Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding);
+ dimensionEncodings[i] = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1]);
+ }
+ }
+
+ return dimensionEncodings;
+ }
+
private TblColRef getTopNNumericColumn(FunctionDesc functionDesc) {
- return functionDesc.getParameter().getColRefs().get(0);
+ if (functionDesc.getParameter().isColumnType() == true) {
+ return functionDesc.getParameter().getColRefs().get(0);
+ }
+ return null;
}
- private TblColRef getTopNLiteralColumn(FunctionDesc functionDesc) {
- return functionDesc.getParameter().getColRefs().get(1);
+ private List<TblColRef> getTopNLiteralColumn(FunctionDesc functionDesc) {
+ List<TblColRef> allColumns = functionDesc.getParameter().getColRefs();
+ if (functionDesc.getParameter().isColumnType() == false) {
+ return allColumns;
+ }
+ return allColumns.subList(1, allColumns.size());
}
private boolean isTopN(FunctionDesc functionDesc) {
return FUNC_TOP_N.equalsIgnoreCase(functionDesc.getExpression());
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 0906d5f..9450555 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.model;
import java.util.*;
+import com.fasterxml.jackson.annotation.JsonInclude;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.measure.basic.BasicMeasureType;
@@ -61,8 +62,9 @@ public class FunctionDesc {
@JsonProperty("returntype")
private String returnType;
- @JsonProperty("configurations")
- private HashMap<String, String> configurations = new LinkedHashMap<String, String>();
+ @JsonProperty("configuration")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private HashMap<String, String> configuration = new LinkedHashMap<String, String>();
private DataType returnDataType;
private MeasureType<?> measureType;
@@ -265,12 +267,12 @@ public class FunctionDesc {
return null;
}
- public HashMap<String, String> getConfigurations() {
- return configurations;
+ public HashMap<String, String> getConfiguration() {
+ return configuration;
}
- public void setConfigurations(HashMap<String, String> configurations) {
- this.configurations = configurations;
+ public void setConfiguration(HashMap<String, String> configurations) {
+ this.configuration = configurations;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index c8ac1a9..dde791c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.model;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
@@ -34,6 +35,7 @@ public class MeasureDesc {
@JsonProperty("function")
private FunctionDesc function;
@JsonProperty("dependent_measure_ref")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
private String dependentMeasureRef;
public String getName() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index a501bd1..ca1b35c 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -156,7 +156,8 @@
"next_parameter" : null
}
},
- "returntype" : "topn(100)"
+ "returntype" : "topn(100)",
+ "configuration": {"topn.encoding.SELLER_ID" : "int:4"}
},
"dependent_measure_ref" : null
}, {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
index a8fe363..f5333b0 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
@@ -38,7 +38,10 @@
"lstg_format_name",
"LSTG_SITE_ID",
"SLR_SEGMENT_CD",
- "seller_id"
+ "TRANS_ID",
+ "CAL_DT",
+ "LEAF_CATEG_ID",
+ "SELLER_ID"
]
},
{
@@ -76,20 +79,12 @@
"cal_dt",
"week_beg_dt"
]
- },
- {
- "table": "default.test_kylin_fact",
- "columns": [
- "TRANS_ID",
- "CAL_DT",
- "LEAF_CATEG_ID",
- "SELLER_ID"
- ]
}
],
"metrics": [
"PRICE",
- "ITEM_COUNT"
+ "ITEM_COUNT",
+ "SELLER_ID"
],
"last_modified" : 1422435345352,
"fact_table" : "DEFAULT.TEST_KYLIN_FACT",
http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
index c85e5d2..8be6dc5 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
@@ -54,10 +54,12 @@
{
"table": "default.test_kylin_fact",
"columns": [
+ "TRANS_ID",
+ "CAL_DT",
"lstg_format_name",
"LSTG_SITE_ID",
"SLR_SEGMENT_CD",
- "seller_id"
+ "SELLER_ID"
]
},
{
@@ -95,20 +97,12 @@
"cal_dt",
"week_beg_dt"
]
- },
- {
- "table": "default.test_kylin_fact",
- "columns": [
- "TRANS_ID",
- "CAL_DT",
- "LEAF_CATEG_ID",
- "SELLER_ID"
- ]
}
],
"metrics": [
"PRICE",
- "ITEM_COUNT"
+ "ITEM_COUNT",
+ "SELLER_ID"
],
"last_modified": 1422435345352,
"fact_table": "DEFAULT.TEST_KYLIN_FACT",
http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/examples/test_case_data/sandbox/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_hive_conf.xml b/examples/test_case_data/sandbox/kylin_hive_conf.xml
index f7ae507..e75f191 100644
--- a/examples/test_case_data/sandbox/kylin_hive_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_hive_conf.xml
@@ -54,18 +54,8 @@
-->
<property>
- <name>hive.merge.mapfiles</name>
- <value>true</value>
- <description>Enable hive file merge on mapper only job</description>
- </property>
- <property>
- <name>hive.merge.mapredfiles</name>
- <value>true</value>
- <description>Enable hive file merge on map-reduce job</description>
- </property>
- <property>
<name>hive.merge.size.per.task</name>
- <value>64000000</value>
+ <value>32000000</value>
<description>Size for the merged file</description>
</property>
</configuration>
\ No newline at end of file