You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/03/04 03:03:16 UTC
[35/43] kylin git commit: Revert "KYLIN-1415 Cube parallel merge"
Revert "KYLIN-1415 Cube parallel merge"
This reverts commit 8aef3cab02bc670dbabb7d1418b8e3a8a394f8fc.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/efd07403
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/efd07403
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/efd07403
Branch: refs/heads/helix-rebase
Commit: efd0740362b610a8fb0b819820a194eede806a32
Parents: 495ad92
Author: shaofengshi <sh...@apache.org>
Authored: Tue Feb 16 10:40:07 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Mar 4 09:52:20 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 48 +++++++-------------
.../apache/kylin/rest/service/CubeService.java | 2 +-
2 files changed, 18 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/efd07403/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 16922ac..84dd30a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -18,11 +18,13 @@
package org.apache.kylin.cube;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.annotation.Nullable;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
@@ -55,11 +57,11 @@ import org.apache.kylin.source.SourceFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
/**
* @author yangli9
@@ -456,14 +458,8 @@ public class CubeManager implements IRealizationProvider {
}
public CubeSegment mergeSegments(CubeInstance cube, final long startDate, final long endDate, boolean forceMergeEmptySeg) throws IOException {
- return mergeSegments(cube, startDate, endDate, forceMergeEmptySeg, true);
- }
-
- public CubeSegment mergeSegments(CubeInstance cube, final long startDate, final long endDate, boolean forceMergeEmptySeg, boolean strictCheck) throws IOException {
+ checkNoBuildingSegment(cube);
checkCubeIsPartitioned(cube);
-
- if (strictCheck)
- checkNoBuildingSegment(cube);
Pair<Long, Long> range = alignMergeRange(cube, startDate, endDate);
CubeSegment newSegment = newSegment(cube, range.getFirst(), range.getSecond());
@@ -623,23 +619,13 @@ public class CubeManager implements IRealizationProvider {
return null;
}
- List<CubeSegment> readySegments = Lists.newArrayList(cube.getSegments(SegmentStatusEnum.READY));
-
- if (readySegments.size() == 0) {
- logger.debug("Cube " + cube.getName() + " has no ready segment to merge");
+ if (cube.getBuildingSegments().size() > 0) {
+ logger.debug("Cube " + cube.getName() + " has bulding segment, will not trigger merge at this moment");
return null;
}
- List<CubeSegment> buildingSegments = Lists.newArrayList(cube.getSegments(SegmentStatusEnum.NEW));
- List<CubeSegment> toSkipSegments = Lists.newArrayList();
- for (CubeSegment building : buildingSegments) {
- for (CubeSegment ready : readySegments) {
- if (ready.getDateRangeStart() >= building.getDateRangeStart() && ready.getDateRangeEnd() <= building.getDateRangeEnd()) {
- toSkipSegments.add(ready);
- }
- }
- }
- readySegments.removeAll(toSkipSegments);
+ List<CubeSegment> readySegments = Lists.newArrayList(cube.getSegments(SegmentStatusEnum.READY));
+
if (readySegments.size() == 0) {
logger.debug("Cube " + cube.getName() + " has no ready segment to merge");
return null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/efd07403/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index abc613b..b2a278a 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -653,7 +653,7 @@ public class CubeService extends BasicService {
cube = getCubeManager().getCube(cubeName);
CubeSegment newSeg = getCubeManager().autoMergeCubeSegments(cube);
if (newSeg != null) {
- newSeg = getCubeManager().mergeSegments(cube, newSeg.getDateRangeStart(), newSeg.getDateRangeEnd(), true, false);
+ newSeg = getCubeManager().mergeSegments(cube, newSeg.getDateRangeStart(), newSeg.getDateRangeEnd(), true);
logger.debug("Will submit merge job on " + newSeg);
DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(newSeg, "SYSTEM");
getExecutableManager().addJob(job);