You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/06/25 05:30:22 UTC
[kylin] branch 2.6.x updated: KYLIN-3994: StorageCleanupJob may
delete cube id data of new built segment because of cube cache in
CubeManager (#633)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/2.6.x by this push:
new 0c1dd10 KYLIN-3994: StorageCleanupJob may delete cube id data of new built segment because of cube cache in CubeManager (#633)
0c1dd10 is described below
commit 0c1dd10fd98062df65b7b55ba46cba1eda0ff1ff
Author: Liu Shaohui <ls...@gmail.com>
AuthorDate: Fri May 31 18:18:19 2019 +0800
KYLIN-3994: StorageCleanupJob may delete cube id data of new built segment because of cube cache in CubeManager (#633)
* KYLIN-3994: StorageCleanupJob may delete cube id data of new built segment because of cube cache in CubeManager
---
.../src/main/java/org/apache/kylin/cube/CubeManager.java | 16 ++++++++++++++++
.../org/apache/kylin/rest/job/MetadataCleanupJob.java | 2 +-
.../apache/kylin/rest/job/StorageCleanJobHbaseUtil.java | 2 +-
.../org/apache/kylin/rest/job/StorageCleanupJob.java | 2 +-
4 files changed, 19 insertions(+), 3 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index bd9832f..db9c095 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -182,12 +182,28 @@ public class CubeManager implements IRealizationProvider {
}
}
+ /**
+ * List all cubes from cache. Note the metadata may be out of date
+ * @return
+ */
public List<CubeInstance> listAllCubes() {
try (AutoLock lock = cubeMapLock.lockForRead()) {
return new ArrayList<CubeInstance>(cubeMap.values());
}
}
+ /**
+ * Reload the cubes from database and list all cubes
+ * @return
+ * @throws IOException
+ */
+ public List<CubeInstance> reloadAndListAllCubes() throws IOException {
+ try (AutoLock lock = cubeMapLock.lockForWrite()) {
+ crud.reloadAll();
+ }
+ return listAllCubes();
+ }
+
public CubeInstance getCube(String cubeName) {
try (AutoLock lock = cubeMapLock.lockForRead()) {
return cubeMap.get(cubeName);
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
index 360818b..17eebd6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
@@ -132,7 +132,7 @@ public class MetadataCleanupJob {
// exclude resources in use
Set<String> activeResources = Sets.newHashSet();
- for (CubeInstance cube : cubeManager.listAllCubes()) {
+ for (CubeInstance cube : cubeManager.reloadAndListAllCubes()) {
activeResources.addAll(cube.getSnapshots().values());
for (CubeSegment segment : cube.getSegments()) {
activeResources.addAll(segment.getSnapshotPaths());
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
index 4c8c426..c29c4de 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
@@ -81,7 +81,7 @@ public class StorageCleanJobHbaseUtil {
}
// remove every segment htable from drop list
- for (CubeInstance cube : cubeMgr.listAllCubes()) {
+ for (CubeInstance cube : cubeMgr.reloadAndListAllCubes()) {
for (CubeSegment seg : cube.getSegments()) {
String tablename = seg.getStorageLocationIdentifier();
if (allTablesNeedToBeDropped.contains(tablename)) {
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
index b73e916..3b2e787 100755
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
@@ -256,7 +256,7 @@ public class StorageCleanupJob extends AbstractApplication {
}
// remove every segment working dir from deletion list
- for (CubeInstance cube : cubeMgr.listAllCubes()) {
+ for (CubeInstance cube : cubeMgr.reloadAndListAllCubes()) {
for (CubeSegment seg : cube.getSegments()) {
String jobUuid = seg.getLastBuildJobID();
if (jobUuid != null && jobUuid.equals("") == false) {