You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/07/25 12:12:41 UTC
[09/50] [abbrv] kylin git commit: KYLIN-1854 Allow deleting cube
instance when its underlying cubedesc went wrong
KYLIN-1854 Allow deleting cube instance when its underlying cubedesc went wrong
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/edfb37d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/edfb37d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/edfb37d0
Branch: refs/heads/1.5.x-HBase1.x
Commit: edfb37d08c94294801307c396b94df103b8706c8
Parents: 4cd733a
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Jul 7 15:31:56 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Jul 7 15:31:56 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/DeployUtil.java | 6 ++--
.../java/org/apache/kylin/cube/CubeManager.java | 15 ++++++---
.../apache/kylin/rest/service/CubeService.java | 32 ++++++++++++++------
3 files changed, 35 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/edfb37d0/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 22fe48a..da97df3 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -32,9 +32,9 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.ResourceTool;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.job.dataGen.FactTableGenerator;
import org.apache.kylin.job.streaming.KafkaDataLoader;
import org.apache.kylin.job.streaming.StreamingTableDataGenerator;
@@ -71,9 +71,7 @@ public class DeployUtil {
// update cube desc signature.
for (CubeInstance cube : CubeManager.getInstance(config()).listAllCubes()) {
- cube.getDescriptor().setSignature(cube.getDescriptor().calculateSignature());
- CubeUpdate cubeBuilder = new CubeUpdate(cube);
- CubeManager.getInstance(config()).updateCube(cubeBuilder);
+ CubeDescManager.getInstance(config()).updateCubeDesc(cube.getDescriptor());//enforce signature updating
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/edfb37d0/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index a200f5d..0941d56 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -810,13 +810,20 @@ public class CubeManager implements IRealizationProvider {
ResourceStore store = getStore();
List<String> paths = store.collectResourceRecursively(ResourceStore.CUBE_RESOURCE_ROOT, ".json");
- logger.debug("Loading Cube from folder " + store.getReadableResourcePath(ResourceStore.CUBE_RESOURCE_ROOT));
+ logger.info("Loading Cube from folder " + store.getReadableResourcePath(ResourceStore.CUBE_RESOURCE_ROOT));
+ int succeed = 0;
+ int fail = 0;
for (String path : paths) {
- reloadCubeLocalAt(path);
+ CubeInstance cube = reloadCubeLocalAt(path);
+ if (cube == null) {
+ fail++;
+ } else {
+ succeed++;
+ }
}
- logger.debug("Loaded " + paths.size() + " Cube(s)");
+ logger.info("Loaded " + succeed + " cubes, fail on " + fail + " cubes");
}
private synchronized CubeInstance reloadCubeLocalAt(String path) {
@@ -855,7 +862,7 @@ public class CubeManager implements IRealizationProvider {
return cubeInstance;
} catch (Exception e) {
- logger.error("Error during load cube instance " + path, e);
+ logger.error("Error during load cube instance, skipping : " + path, e);
return null;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/edfb37d0/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index a9d4bfc..72942e8 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -282,8 +282,15 @@ public class CubeService extends BasicService {
throw new JobException("The cube " + cube.getName() + " has running job, please discard it and try again.");
}
- this.releaseAllSegments(cube);
- getCubeManager().dropCube(cube.getName(), true);
+ try {
+ this.releaseAllJobs(cube);
+ } catch (Exception e) {
+ logger.error("error when releasing all jobs", e);
+ //ignore the exception
+ }
+
+ int cubeNum = getCubeManager().getCubesByDesc(cube.getDescriptor().getName()).size();
+ getCubeManager().dropCube(cube.getName(), cubeNum == 1);//only delete cube desc when no other cube is using it
accessService.clean(cube, true);
}
@@ -550,13 +557,7 @@ public class CubeService extends BasicService {
return CubeManager.getInstance(getConfig()).updateCube(update);
}
- /**
- * purge the cube
- *
- * @throws IOException
- * @throws JobException
- */
- private CubeInstance releaseAllSegments(CubeInstance cube) throws IOException, JobException {
+ private void releaseAllJobs(CubeInstance cube) {
final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null);
for (CubingJob cubingJob : cubingJobs) {
final ExecutableState status = cubingJob.getStatus();
@@ -564,9 +565,20 @@ public class CubeService extends BasicService {
getExecutableManager().discardJob(cubingJob.getId());
}
}
+ }
+
+ /**
+ * purge the cube
+ *
+ * @throws IOException
+ * @throws JobException
+ */
+ private void releaseAllSegments(CubeInstance cube) throws IOException, JobException {
+ releaseAllJobs(cube);
+
CubeUpdate update = new CubeUpdate(cube);
update.setToRemoveSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()]));
- return CubeManager.getInstance(getConfig()).updateCube(update);
+ CubeManager.getInstance(getConfig()).updateCube(update);
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)