You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/21 01:28:51 UTC
[18/25] incubator-kylin git commit: KYLIN-642 Couldn't refresh cube
segment
KYLIN-642 Couldn't refresh cube segment
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/79fe5a36
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/79fe5a36
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/79fe5a36
Branch: refs/heads/master
Commit: 79fe5a365f480cc712a8ab7a2fbf3566fa3fcbc1
Parents: 4f6869e
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Tue Mar 17 11:15:04 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Tue Mar 17 11:15:04 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 50 +++++----
.../apache/kylin/rest/service/JobService.java | 110 +++++++++----------
webapp/app/js/controllers/cubes.js | 2 +-
3 files changed, 81 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/79fe5a36/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 8493c59..75f984c 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -18,36 +18,19 @@
package org.apache.kylin.cube;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.kylin.cube.model.CubeBuildTypeEnum;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.metadata.realization.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.common.restclient.Broadcaster;
import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
@@ -61,6 +44,13 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
/**
@@ -254,7 +244,7 @@ public class CubeManager implements IRealizationProvider {
public Pair<CubeSegment, CubeSegment> appendAndMergeSegments(CubeInstance cube, long endDate) throws IOException {
checkNoBuildingSegment(cube);
checkCubeIsPartitioned(cube);
-
+
if (cube.getSegments().size() == 0)
throw new IllegalStateException("expect at least one existing segment");
@@ -292,6 +282,18 @@ public class CubeManager implements IRealizationProvider {
return newSegment;
}
+
+ public CubeSegment refreshSegment(CubeInstance cube, long startDate, long endDate) throws IOException {
+ checkNoBuildingSegment(cube);
+
+ CubeSegment newSegment = newSegment(cube, startDate, endDate);
+ cube.getSegments().add(newSegment);
+ Collections.sort(cube.getSegments());
+ updateCube(cube);
+
+ return newSegment;
+ }
+
public CubeSegment mergeSegments(CubeInstance cube, final long startDate, final long endDate) throws IOException {
checkNoBuildingSegment(cube);
checkCubeIsPartitioned(cube);
@@ -509,7 +511,7 @@ public class CubeManager implements IRealizationProvider {
/**
* Smartly figure out the TOBE segments once all new segments are built.
- * - Ensures no gap, no overlap
+ * - Ensures no gap, no overlap
* - Favors new segments over the old
* - Favors big segments over the small
*/
@@ -533,7 +535,7 @@ public class CubeManager implements IRealizationProvider {
}
firstSeg.validate();
- for (int i = 0, j = 1; j < tobe.size();) {
+ for (int i = 0, j = 1; j < tobe.size(); ) {
CubeSegment is = tobe.get(i);
CubeSegment js = tobe.get(j);
js.validate();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/79fe5a36/server/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
index 1b89f87..05031b9 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -18,25 +18,12 @@
package org.apache.kylin.rest.service;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.kylin.rest.constant.Constant;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.security.access.prepost.PreAuthorize;
-import org.springframework.stereotype.Component;
-
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeBuildTypeEnum;
@@ -50,10 +37,18 @@ import org.apache.kylin.job.cube.CubingJob;
import org.apache.kylin.job.cube.CubingJobBuilder;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.exception.JobException;
+import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.Output;
-import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.rest.constant.Constant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.util.*;
/**
* @author ysong1
@@ -105,20 +100,20 @@ public class JobService extends BasicService {
private ExecutableState parseToExecutableState(JobStatusEnum status) {
switch (status) {
- case DISCARDED:
- return ExecutableState.DISCARDED;
- case ERROR:
- return ExecutableState.ERROR;
- case FINISHED:
- return ExecutableState.SUCCEED;
- case NEW:
- return ExecutableState.READY;
- case PENDING:
- return ExecutableState.READY;
- case RUNNING:
- return ExecutableState.RUNNING;
- default:
- throw new RuntimeException("illegal status:" + status);
+ case DISCARDED:
+ return ExecutableState.DISCARDED;
+ case ERROR:
+ return ExecutableState.ERROR;
+ case FINISHED:
+ return ExecutableState.SUCCEED;
+ case NEW:
+ return ExecutableState.READY;
+ case PENDING:
+ return ExecutableState.READY;
+ case RUNNING:
+ return ExecutableState.RUNNING;
+ default:
+ throw new RuntimeException("illegal status:" + status);
}
}
@@ -135,7 +130,7 @@ public class JobService extends BasicService {
CubingJob job;
CubingJobBuilder builder = new CubingJobBuilder(new JobEngineConfig(getConfig()));
builder.setSubmitter(submitter);
-
+
if (buildType == CubeBuildTypeEnum.BUILD) {
if (cube.getDescriptor().hasHolisticCountDistinctMeasures() && cube.getSegments().size() > 0) {
Pair<CubeSegment, CubeSegment> segs = getCubeManager().appendAndMergeSegments(cube, endDate);
@@ -147,6 +142,9 @@ public class JobService extends BasicService {
} else if (buildType == CubeBuildTypeEnum.MERGE) {
CubeSegment newSeg = getCubeManager().mergeSegments(cube, startDate, endDate);
job = builder.mergeJob(newSeg);
+ } else if (buildType == CubeBuildTypeEnum.REFRESH) {
+ CubeSegment refreshSeg = getCubeManager().refreshSegment(cube, startDate, endDate);
+ job = builder.buildJob(refreshSeg);
} else {
throw new JobException("invalid build type:" + buildType);
}
@@ -215,37 +213,37 @@ public class JobService extends BasicService {
private JobStatusEnum parseToJobStatus(ExecutableState state) {
switch (state) {
- case READY:
- return JobStatusEnum.PENDING;
- case RUNNING:
- return JobStatusEnum.RUNNING;
- case ERROR:
- return JobStatusEnum.ERROR;
- case DISCARDED:
- return JobStatusEnum.DISCARDED;
- case SUCCEED:
- return JobStatusEnum.FINISHED;
- case STOPPED:
- default:
- throw new RuntimeException("invalid state:" + state);
+ case READY:
+ return JobStatusEnum.PENDING;
+ case RUNNING:
+ return JobStatusEnum.RUNNING;
+ case ERROR:
+ return JobStatusEnum.ERROR;
+ case DISCARDED:
+ return JobStatusEnum.DISCARDED;
+ case SUCCEED:
+ return JobStatusEnum.FINISHED;
+ case STOPPED:
+ default:
+ throw new RuntimeException("invalid state:" + state);
}
}
private JobStepStatusEnum parseToJobStepStatus(ExecutableState state) {
switch (state) {
- case READY:
- return JobStepStatusEnum.PENDING;
- case RUNNING:
- return JobStepStatusEnum.RUNNING;
- case ERROR:
- return JobStepStatusEnum.ERROR;
- case DISCARDED:
- return JobStepStatusEnum.DISCARDED;
- case SUCCEED:
- return JobStepStatusEnum.FINISHED;
- case STOPPED:
- default:
- throw new RuntimeException("invalid state:" + state);
+ case READY:
+ return JobStepStatusEnum.PENDING;
+ case RUNNING:
+ return JobStepStatusEnum.RUNNING;
+ case ERROR:
+ return JobStepStatusEnum.ERROR;
+ case DISCARDED:
+ return JobStepStatusEnum.DISCARDED;
+ case SUCCEED:
+ return JobStepStatusEnum.FINISHED;
+ case STOPPED:
+ default:
+ throw new RuntimeException("invalid state:" + state);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/79fe5a36/webapp/app/js/controllers/cubes.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js
index df86816..a08422a 100755
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -339,7 +339,7 @@ KylinApp
return cube;
},
buildType: function () {
- return 'BUILD';
+ return 'REFRESH';
}
}
});