You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/06/11 05:28:38 UTC
[kylin] branch master updated: KYLIN-3115 Incompatible
RowKeySplitter initialize between build and merge job
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new f6b1dfb KYLIN-3115 Incompatible RowKeySplitter initialize between build and merge job
f6b1dfb is described below
commit f6b1dfb5ef3239ea252b1498bf4c51235361bbcd
Author: shaofengshi <sh...@apache.org>
AuthorDate: Wed Jun 6 22:59:30 2018 +0800
KYLIN-3115 Incompatible RowKeySplitter initialize between build and merge job
---
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java | 3 +++
core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java | 2 +-
.../java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java | 6 ++----
.../kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java | 2 +-
.../java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java | 2 +-
.../main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java | 2 +-
.../apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java | 2 +-
.../main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java | 2 +-
9 files changed, 15 insertions(+), 10 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b2d8e99..6014b1a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -484,6 +484,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.cube.rowkey.max-size", "63"));
}
+ public int getDimensionEncodingMaxLength() {
+ return Integer.parseInt(getOptional("kylin.metadata.dimension-encoding-max-length", "256"));
+ }
+
public int getMaxBuildingSegments() {
return Integer.parseInt(getOptional("kylin.cube.max-building-segments", "10"));
}
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
index 0c54ecf..d0015a7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
@@ -63,6 +63,9 @@ public class RowKeySplitter implements java.io.Serializable {
return bufferSize;
}
+ public RowKeySplitter(CubeSegment cubeSeg) {
+ this(cubeSeg, cubeSeg.getCubeDesc().getRowkey().getRowKeyColumns().length + 2, cubeSeg.getConfig().getDimensionEncodingMaxLength());
+ }
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
this.cubeSegment = cubeSeg;
this.enableSharding = cubeSeg.isEnableSharding();
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
index bb03c4c..923ddb8 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
@@ -45,7 +45,7 @@ public class RowKeyDecoder {
public RowKeyDecoder(CubeSegment cubeSegment) {
this.cubeDesc = cubeSegment.getCubeDesc();
- this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 255);
+ this.rowKeySplitter = new RowKeySplitter(cubeSegment);
this.colIO = new RowKeyColumnIO(cubeSegment.getDimensionEncodingMap());
this.values = new ArrayList<String>();
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
index f1b40de..4d9885e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
@@ -47,15 +47,13 @@ public class NDCuboidBuilder implements Serializable {
private ByteArray newKeyBuf = ByteArray.allocate(RowConstants.ROWKEY_BUFFER_SIZE);
public NDCuboidBuilder(CubeSegment cubeSegment) {
- this.cubeSegment = cubeSegment;
- this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
- this.rowKeyEncoderProvider = new RowKeyEncoderProvider(cubeSegment);
+ this(cubeSegment, new RowKeyEncoderProvider(cubeSegment));
}
public NDCuboidBuilder(CubeSegment cubeSegment, RowKeyEncoderProvider rowKeyEncoderProvider) {
this.cubeSegment = cubeSegment;
this.rowKeyEncoderProvider = rowKeyEncoderProvider;
- this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
+ this.rowKeySplitter = new RowKeySplitter(cubeSegment);
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java
index d8ae3ab..2bb8349 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java
@@ -64,7 +64,7 @@ public class FilterRecommendCuboidDataMapper extends KylinMapper<Text, Text, Tex
CubeSegment optSegment = cube.getSegmentById(segmentID);
CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(optSegment);
- rowKeySplitter = new RowKeySplitter(originalSegment, 65, 255);
+ rowKeySplitter = new RowKeySplitter(originalSegment);
baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
recommendCuboids = cube.getCuboidsRecommend();
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index 255315a..0283c21 100755
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -110,7 +110,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
IMROutput2.IMRMergeOutputFormat outputFormat = MRUtil.getBatchMergeOutputSide2(mergedCubeSegment).getOuputFormat();
sourceCubeSegment = outputFormat.findSourceSegment(fileSplit, cube);
- rowKeySplitter = new RowKeySplitter(sourceCubeSegment, 65, 255);
+ rowKeySplitter = new RowKeySplitter(sourceCubeSegment);
rowKeyEncoderProvider = new RowKeyEncoderProvider(mergedCubeSegment);
measureDescs = cubeDesc.getMeasures();
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
index a58415a..f2ec34c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
@@ -78,7 +78,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
ndCuboidBuilder = new NDCuboidBuilder(cubeSegment);
// initialize CubiodScheduler
cuboidScheduler = CuboidSchedulerUtil.getCuboidSchedulerByMode(cubeSegment, cuboidModeName);
- rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
+ rowKeySplitter = new RowKeySplitter(cubeSegment);
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java
index 58b553e..35b8924 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java
@@ -79,7 +79,7 @@ public class UpdateOldCuboidShardMapper extends KylinMapper<Text, Text, Text, Te
cubeDesc = cube.getDescriptor();
baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
- rowKeySplitter = new RowKeySplitter(oldSegment, 65, 256);
+ rowKeySplitter = new RowKeySplitter(oldSegment);
rowKeyEncoderProvider = new RowKeyEncoderProvider(cubeSegment);
}
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index d609edb..8f547fd 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -438,7 +438,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
this.cubeDesc = cubeInstance.getDescriptor();
this.cuboidScheduler = cubeSegment.getCuboidScheduler();
this.ndCuboidBuilder = new NDCuboidBuilder(cubeSegment, new RowKeyEncoderProvider(cubeSegment));
- this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
+ this.rowKeySplitter = new RowKeySplitter(cubeSegment);
}
@Override
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.