You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/11/27 05:57:34 UTC
incubator-kylin git commit: Incease 4096 column length limit to 16384
in BaseCuboidMapperBase
Repository: incubator-kylin
Updated Branches:
refs/heads/2.x-staging f03a256cf -> 6515b0a02
Incease 4096 column length limit to 16384 in BaseCuboidMapperBase
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/6515b0a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/6515b0a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/6515b0a0
Branch: refs/heads/2.x-staging
Commit: 6515b0a023912156548c8cf0fb4f60d42de86738
Parents: f03a256
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Nov 27 12:57:19 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Nov 27 12:57:19 2015 +0800
----------------------------------------------------------------------
.../engine/mr/steps/BaseCuboidMapperBase.java | 40 ++++++++++----------
1 file changed, 19 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6515b0a0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 44311c2..557e325 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -83,7 +83,7 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
intermediateTableDesc = new CubeJoinedFlatTableDesc(cube.getDescriptor(), cubeSegment);
- bytesSplitter = new BytesSplitter(200, 4096);
+ bytesSplitter = new BytesSplitter(200, 16384);
rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);
measureCodec = new MeasureCodec(cubeDesc.getMeasures());
@@ -95,7 +95,7 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
initTopNLiteralColDictionaryMap();
initNullBytes();
}
-
+
private void initTopNLiteralColDictionaryMap() {
topNLiteralColDictMap = Maps.newHashMap();
for (int measureIdx = 0; measureIdx < measures.length; measureIdx++) {
@@ -105,7 +105,7 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[measureIdx];
int literalColIdx = flatTableIdx[flatTableIdx.length - 1];
TblColRef literalCol = func.getTopNLiteralColumn();
- Dictionary<String> dictionary = (Dictionary<String>)cubeSegment.getDictionary(literalCol);
+ Dictionary<String> dictionary = (Dictionary<String>) cubeSegment.getDictionary(literalCol);
topNLiteralColDictMap.put(literalColIdx, dictionary);
}
}
@@ -164,14 +164,14 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
// constant
if (flatTableIdx == null) {
result = Bytes.toBytes(paramDesc.getValue());
- }
+ }
// count and count distinct
else if (func.isCount() || func.isHolisticCountDistinct()) {
// note for holistic count distinct, this value will be ignored
result = ONE;
}
// topN, need encode the key column
- else if(func.isTopN()) {
+ else if (func.isTopN()) {
// encode the key column with dict, and get the counter column;
int keyColIndex = flatTableIdx[flatTableIdx.length - 1];
Dictionary<String> literalColDict = topNLiteralColDictMap.get(keyColIndex);
@@ -186,27 +186,25 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
// get the counter column value
valueBuf.putDouble(Double.valueOf(Bytes.toString(splitBuffers[flatTableIdx[0]].value)));
}
-
+
result = valueBuf.array();
-
- }
+
+ }
// normal case, concat column values
else {
- // for multiple columns, their values are joined
- for (int i = 0; i < flatTableIdx.length; i++) {
- SplittedBytes split = splitBuffers[flatTableIdx[i]];
- if (result == null) {
- result = Arrays.copyOf(split.value, split.length);
- } else {
- byte[] newResult = new byte[result.length + split.length];
- System.arraycopy(result, 0, newResult, 0, result.length);
- System.arraycopy(split.value, 0, newResult, result.length, split.length);
- result = newResult;
- }
+ // for multiple columns, their values are joined
+ for (int i = 0; i < flatTableIdx.length; i++) {
+ SplittedBytes split = splitBuffers[flatTableIdx[i]];
+ if (result == null) {
+ result = Arrays.copyOf(split.value, split.length);
+ } else {
+ byte[] newResult = new byte[result.length + split.length];
+ System.arraycopy(result, 0, newResult, 0, result.length);
+ System.arraycopy(split.value, 0, newResult, result.length, split.length);
+ result = newResult;
}
}
-
-
+ }
if (isNull(result)) {
result = null;