You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/06/30 09:50:32 UTC
incubator-kylin git commit: KYLIN-863 add parameter for enforce merge
empty segment
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8 4f2ebb5ae -> 1d0803023
KYLIN-863 add parameter for enforce merge empty segment
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1d080302
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1d080302
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1d080302
Branch: refs/heads/0.8
Commit: 1d080302329cbcae4ed78ba40cce1322781efb0e
Parents: 4f2ebb5
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jun 30 15:50:16 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Jun 30 15:50:16 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 17 ++++++++++++++++-
.../kylin/rest/controller/CubeController.java | 2 +-
.../apache/kylin/rest/request/JobBuildRequest.java | 9 +++++++++
.../org/apache/kylin/rest/service/JobService.java | 4 ++--
4 files changed, 28 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1d080302/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 76d57c7..ff4a0ae 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -418,13 +418,28 @@ public class CubeManager implements IRealizationProvider {
return newSegment;
}
- public CubeSegment mergeSegments(CubeInstance cube, final long startDate, final long endDate) throws IOException {
+ public CubeSegment mergeSegments(CubeInstance cube, final long startDate, final long endDate, boolean forceMergeEmptySeg) throws IOException {
checkNoBuildingSegment(cube);
checkCubeIsPartitioned(cube);
Pair<Long, Long> range = alignMergeRange(cube, startDate, endDate);
CubeSegment newSegment = newSegment(cube, range.getFirst(), range.getSecond());
+ List<CubeSegment> mergingSegments = cube.getMergingSegments(newSegment);
+
+ if (forceMergeEmptySeg == false) {
+ List<String> emptySegment = Lists.newArrayList();
+ for (CubeSegment seg : mergingSegments) {
+ if (seg.getSizeKB() == 0) {
+ emptySegment.add(seg.getName());
+ }
+ }
+
+ if (emptySegment.size() > 0) {
+ throw new IllegalArgumentException("Empty cube segment found, couldn't merge unless 'forceMergeEmptySegment' set to true: " + emptySegment);
+ }
+ }
+
validateNewSegments(cube, false, newSegment);
CubeBuilder cubeBuilder = new CubeBuilder(cube);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1d080302/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 441c26b..53ba6b6 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -188,7 +188,7 @@ public class CubeController extends BasicController {
String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
CubeInstance cube = jobService.getCubeManager().getCube(cubeName);
return jobService.submitJob(cube, jobBuildRequest.getStartTime(), jobBuildRequest.getEndTime(), //
- CubeBuildTypeEnum.valueOf(jobBuildRequest.getBuildType()), submitter);
+ CubeBuildTypeEnum.valueOf(jobBuildRequest.getBuildType()), jobBuildRequest.isForceMergeEmptySegment(), submitter);
} catch (JobException e) {
logger.error(e.getLocalizedMessage(), e);
throw new InternalErrorException(e.getLocalizedMessage());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1d080302/server/src/main/java/org/apache/kylin/rest/request/JobBuildRequest.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/request/JobBuildRequest.java b/server/src/main/java/org/apache/kylin/rest/request/JobBuildRequest.java
index 0e06771..f7ec88e 100644
--- a/server/src/main/java/org/apache/kylin/rest/request/JobBuildRequest.java
+++ b/server/src/main/java/org/apache/kylin/rest/request/JobBuildRequest.java
@@ -30,6 +30,8 @@ public class JobBuildRequest {
private String buildType;
+ private boolean forceMergeEmptySegment = false;
+
public long getStartTime() {
return startTime;
}
@@ -54,4 +56,11 @@ public class JobBuildRequest {
this.buildType = buildType;
}
+ public boolean isForceMergeEmptySegment() {
+ return forceMergeEmptySegment;
+ }
+
+ public void setForceMergeEmptySegment(boolean forceMergeEmptySegment) {
+ this.forceMergeEmptySegment = forceMergeEmptySegment;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1d080302/server/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
index 02e0527..08c17ef 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -124,7 +124,7 @@ public class JobService extends BasicService {
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
- public JobInstance submitJob(CubeInstance cube, long startDate, long endDate, CubeBuildTypeEnum buildType, String submitter) throws IOException, JobException {
+ public JobInstance submitJob(CubeInstance cube, long startDate, long endDate, CubeBuildTypeEnum buildType, boolean forceMergeEmptySeg, String submitter) throws IOException, JobException {
final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.allOf(ExecutableState.class));
for (CubingJob job : cubingJobs) {
@@ -146,7 +146,7 @@ public class JobService extends BasicService {
job = builder.buildJob(newSeg);
}
} else if (buildType == CubeBuildTypeEnum.MERGE) {
- CubeSegment newSeg = getCubeManager().mergeSegments(cube, startDate, endDate);
+ CubeSegment newSeg = getCubeManager().mergeSegments(cube, startDate, endDate, forceMergeEmptySeg);
job = builder.mergeJob(newSeg);
} else if (buildType == CubeBuildTypeEnum.REFRESH) {
CubeSegment refreshSeg = getCubeManager().refreshSegment(cube, startDate, endDate);