You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/07/07 04:57:50 UTC
[kylin] 01/01: KYLIN-3370 bug fix in UpdateCubeInfoAfterMergeStep
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch sync
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a08fed5b5273c16abfaf6a7a09ba44545069e88a
Author: Li Yang <li...@apache.org>
AuthorDate: Thu Jul 5 14:16:01 2018 +0800
KYLIN-3370 bug fix in UpdateCubeInfoAfterMergeStep
---
.../engine/mr/steps/UpdateCubeInfoAfterMergeStep.java | 18 +++++++++---------
.../mr/steps/UpdateCubeInfoAfterOptimizeStep.java | 1 +
.../apache/kylin/provision/BuildCubeWithEngine.java | 1 +
3 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
index 56f5077..4cf6fc6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
@@ -63,28 +63,29 @@ public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
if (mergingSegmentIds.isEmpty()) {
return ExecuteResult.createFailed(new SegmentNotFoundException("there are no merging segments"));
}
+
long sourceCount = 0L;
long sourceSize = 0L;
-
boolean isOffsetCube = mergedSegment.isOffsetCube();
Long tsStartMin = Long.MAX_VALUE, tsEndMax = 0L;
+ for (String id : mergingSegmentIds) {
+ CubeSegment segment = cube.getSegmentById(id);
+ sourceCount += segment.getInputRecords();
+ sourceSize += segment.getInputRecordsSize();
+ tsStartMin = Math.min(tsStartMin, segment.getTSRange().start.v);
+ tsEndMax = Math.max(tsEndMax, segment.getTSRange().end.v);
+ }
+
Map<String, DimensionRangeInfo> mergedSegDimRangeMap = null;
for (String id : mergingSegmentIds) {
CubeSegment segment = cube.getSegmentById(id);
Map<String, DimensionRangeInfo> segDimRangeMap = segment.getDimensionRangeInfoMap();
- if (segDimRangeMap.isEmpty()) {
- continue;
- }
if (mergedSegDimRangeMap == null) {
mergedSegDimRangeMap = segDimRangeMap;
} else {
mergedSegDimRangeMap = DimensionRangeInfo.mergeRangeMap(cube.getModel(), segDimRangeMap,
mergedSegDimRangeMap);
}
- sourceCount += segment.getInputRecords();
- sourceSize += segment.getInputRecordsSize();
- tsStartMin = Math.min(tsStartMin, segment.getTSRange().start.v);
- tsEndMax = Math.max(tsEndMax, segment.getTSRange().end.v);
}
// update segment info
@@ -94,7 +95,6 @@ public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
mergedSegment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
mergedSegment.setLastBuildTime(System.currentTimeMillis());
mergedSegment.setDimensionRangeInfoMap(mergedSegDimRangeMap);
-
if (isOffsetCube) {
SegmentRange.TSRange tsRange = new SegmentRange.TSRange(tsStartMin, tsEndMax);
mergedSegment.setTSRange(tsRange);
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterOptimizeStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterOptimizeStep.java
index d013386..03aa616 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterOptimizeStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterOptimizeStep.java
@@ -59,6 +59,7 @@ public class UpdateCubeInfoAfterOptimizeStep extends AbstractExecutable {
segment.setSizeKB(cubeSizeBytes / 1024);
segment.setInputRecords(sourceCount);
segment.setInputRecordsSize(sourceSizeBytes);
+ segment.setDimensionRangeInfoMap(originalSegment.getDimensionRangeInfoMap());
try {
cubeManager.promoteNewlyOptimizeSegments(cube, segment);
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 2f22bd4..16ceede 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -313,6 +313,7 @@ public class BuildCubeWithEngine {
checkNormalSegRangeInfo(cubeManager.getCube(cubeName));
if (!buildSegment(cubeName, date2, date3))
return false;
+ checkNormalSegRangeInfo(cubeManager.getCube(cubeName));
if (!optimizeCube(cubeName))
return false;
checkNormalSegRangeInfo(cubeManager.getCube(cubeName));