You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/06/12 15:02:06 UTC
[kylin] branch 2.3.x updated (2e212ae -> d27e94a)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 2e212ae KYLIN-3093 Upgrade curator to 2.12
new 4c0ca9f KYLIN-3115 Incompatible RowKeySplitter initialize between build and merge job
new d27e94a KYLIN-3388 remove create redistribute flat hive table step if not set distribute column
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java | 3 +++
core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java | 2 +-
core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java | 6 +-----
.../java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java | 6 ++----
.../kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java | 2 +-
.../java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java | 2 +-
.../main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java | 2 +-
.../apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java | 1 -
.../apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java | 2 +-
.../main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java | 2 +-
.../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 4 +++-
12 files changed, 19 insertions(+), 17 deletions(-)
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 02/02: KYLIN-3388 remove create redistribute flat hive
table step if not set distribute column
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d27e94a8edd4f8b70c3ff7fd1e8c40ba935789fb
Author: Chao Long <wa...@qq.com>
AuthorDate: Mon Jun 11 21:53:20 2018 +0800
KYLIN-3388 remove create redistribute flat hive table step if not set distribute column
---
core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java | 6 +-----
.../apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java | 1 -
.../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 4 +++-
3 files changed, 4 insertions(+), 7 deletions(-)
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 9612c7e..c97daa7 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -200,11 +200,7 @@ public class JoinedFlatTable {
}
private static void appendDistributeStatement(StringBuilder sql, TblColRef redistCol) {
- if (redistCol != null) {
- sql.append(" DISTRIBUTE BY ").append(colName(redistCol)).append(";\n");
- } else {
- sql.append(" DISTRIBUTE BY RAND()").append(";\n");
- }
+ sql.append(" DISTRIBUTE BY ").append(colName(redistCol)).append(";\n");
}
private static void appendClusterStatement(StringBuilder sql, TblColRef clusterCol) {
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
index 2e3e69e..e7aec8c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
@@ -21,7 +21,6 @@ package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
import java.util.List;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 38656cf..79fb281 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -151,7 +151,9 @@ public class HiveMRInput implements IMRInput {
// then count and redistribute
if (cubeConfig.isHiveRedistributeEnabled()) {
- jobFlow.addTask(createRedistributeFlatHiveTableStep(hiveInitStatements, cubeName));
+ if (flatDesc.getClusterBy() != null || flatDesc.getDistributedBy() != null) {
+ jobFlow.addTask(createRedistributeFlatHiveTableStep(hiveInitStatements, cubeName));
+ }
}
// special for hive
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 01/02: KYLIN-3115 Incompatible RowKeySplitter initialize
between build and merge job
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4c0ca9fea69259943eccfdd6e011a86a3775bc23
Author: shaofengshi <sh...@apache.org>
AuthorDate: Wed Jun 6 22:59:30 2018 +0800
KYLIN-3115 Incompatible RowKeySplitter initialize between build and merge job
---
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java | 3 +++
core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java | 2 +-
.../java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java | 6 ++----
.../kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java | 2 +-
.../java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java | 2 +-
.../main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java | 2 +-
.../apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java | 2 +-
.../main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java | 2 +-
9 files changed, 15 insertions(+), 10 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index eee2d09..55e4faa 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -471,6 +471,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.cube.rowkey.max-size", "63"));
}
+ public int getDimensionEncodingMaxLength() {
+ return Integer.parseInt(getOptional("kylin.metadata.dimension-encoding-max-length", "256"));
+ }
+
public int getMaxBuildingSegments() {
return Integer.parseInt(getOptional("kylin.cube.max-building-segments", "10"));
}
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
index 0c54ecf..d0015a7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
@@ -63,6 +63,9 @@ public class RowKeySplitter implements java.io.Serializable {
return bufferSize;
}
+ public RowKeySplitter(CubeSegment cubeSeg) {
+ this(cubeSeg, cubeSeg.getCubeDesc().getRowkey().getRowKeyColumns().length + 2, cubeSeg.getConfig().getDimensionEncodingMaxLength());
+ }
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
this.cubeSegment = cubeSeg;
this.enableSharding = cubeSeg.isEnableSharding();
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
index bb03c4c..923ddb8 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyDecoder.java
@@ -45,7 +45,7 @@ public class RowKeyDecoder {
public RowKeyDecoder(CubeSegment cubeSegment) {
this.cubeDesc = cubeSegment.getCubeDesc();
- this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 255);
+ this.rowKeySplitter = new RowKeySplitter(cubeSegment);
this.colIO = new RowKeyColumnIO(cubeSegment.getDimensionEncodingMap());
this.values = new ArrayList<String>();
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
index f1b40de..4d9885e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
@@ -47,15 +47,13 @@ public class NDCuboidBuilder implements Serializable {
private ByteArray newKeyBuf = ByteArray.allocate(RowConstants.ROWKEY_BUFFER_SIZE);
public NDCuboidBuilder(CubeSegment cubeSegment) {
- this.cubeSegment = cubeSegment;
- this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
- this.rowKeyEncoderProvider = new RowKeyEncoderProvider(cubeSegment);
+ this(cubeSegment, new RowKeyEncoderProvider(cubeSegment));
}
public NDCuboidBuilder(CubeSegment cubeSegment, RowKeyEncoderProvider rowKeyEncoderProvider) {
this.cubeSegment = cubeSegment;
this.rowKeyEncoderProvider = rowKeyEncoderProvider;
- this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
+ this.rowKeySplitter = new RowKeySplitter(cubeSegment);
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java
index d8ae3ab..2bb8349 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataMapper.java
@@ -64,7 +64,7 @@ public class FilterRecommendCuboidDataMapper extends KylinMapper<Text, Text, Tex
CubeSegment optSegment = cube.getSegmentById(segmentID);
CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(optSegment);
- rowKeySplitter = new RowKeySplitter(originalSegment, 65, 255);
+ rowKeySplitter = new RowKeySplitter(originalSegment);
baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
recommendCuboids = cube.getCuboidsRecommend();
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index 255315a..0283c21 100755
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -110,7 +110,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
IMROutput2.IMRMergeOutputFormat outputFormat = MRUtil.getBatchMergeOutputSide2(mergedCubeSegment).getOuputFormat();
sourceCubeSegment = outputFormat.findSourceSegment(fileSplit, cube);
- rowKeySplitter = new RowKeySplitter(sourceCubeSegment, 65, 255);
+ rowKeySplitter = new RowKeySplitter(sourceCubeSegment);
rowKeyEncoderProvider = new RowKeyEncoderProvider(mergedCubeSegment);
measureDescs = cubeDesc.getMeasures();
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
index a58415a..f2ec34c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
@@ -78,7 +78,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
ndCuboidBuilder = new NDCuboidBuilder(cubeSegment);
// initialize CubiodScheduler
cuboidScheduler = CuboidSchedulerUtil.getCuboidSchedulerByMode(cubeSegment, cuboidModeName);
- rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
+ rowKeySplitter = new RowKeySplitter(cubeSegment);
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java
index 58b553e..35b8924 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardMapper.java
@@ -79,7 +79,7 @@ public class UpdateOldCuboidShardMapper extends KylinMapper<Text, Text, Text, Te
cubeDesc = cube.getDescriptor();
baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
- rowKeySplitter = new RowKeySplitter(oldSegment, 65, 256);
+ rowKeySplitter = new RowKeySplitter(oldSegment);
rowKeyEncoderProvider = new RowKeyEncoderProvider(cubeSegment);
}
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 714991d..da2c52f 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -416,7 +416,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
this.cubeDesc = cubeInstance.getDescriptor();
this.cuboidScheduler = cubeSegment.getCuboidScheduler();
this.ndCuboidBuilder = new NDCuboidBuilder(cubeSegment, new RowKeyEncoderProvider(cubeSegment));
- this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
+ this.rowKeySplitter = new RowKeySplitter(cubeSegment);
}
@Override
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.