You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kylin.apache.org by GitBox <gi...@apache.org> on 2018/12/24 03:15:53 UTC
[GitHub] shaofengshi closed pull request #408: KYLIN-3655 reinitialize
CubeInstance when clear segments
shaofengshi closed pull request #408: KYLIN-3655 reinitialize CubeInstance when clear segments
URL: https://github.com/apache/kylin/pull/408
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 97e319a4df..a022f1ec00 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -256,6 +256,23 @@ public CubeInstance createCube(CubeInstance cube, String projectName, String own
}
}
+ /**
+ * when clear all segments, it's supposed to reinitialize the CubeInstance
+ */
+ public CubeInstance clearSegments(CubeInstance cube) throws IOException {
+ try (AutoLock lock = cubeMapLock.lockForWrite()) {
+ cube = cube.latestCopyForWrite(); // get a latest copy
+ CubeUpdate update = new CubeUpdate(cube);
+ update.setToRemoveSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()]));
+ update.setCuboids(Maps.<Long, Long> newHashMap());
+ update.setCuboidsRecommend(Sets.<Long> newHashSet());
+ update.setUpdateTableSnapshotPath(Maps.<String, String> newHashMap());
+ update.setCreateTimeUTC(System.currentTimeMillis());
+ update.setCuboidLastOptimized(0L);
+ return updateCube(update);
+ }
+ }
+
// try minimize the use of this method, use udpateCubeXXX() instead
public CubeInstance updateCube(CubeUpdate update) throws IOException {
try (AutoLock lock = cubeMapLock.lockForWrite()) {
@@ -394,6 +411,14 @@ private void setCubeMember(CubeInstance cube, CubeUpdate update) {
cube.putSnapshotResPath(lookupSnapshotPathEntry.getKey(), lookupSnapshotPathEntry.getValue());
}
}
+
+ if (update.getCreateTimeUTC() >= 0) {
+ cube.setCreateTimeUTC(update.getCreateTimeUTC());
+ }
+
+ if (update.getCuboidLastOptimized() >= 0) {
+ cube.setCuboidLastOptimized(update.getCuboidLastOptimized());
+ }
}
private void processToUpdateSegments(CubeUpdate update, Segments<CubeSegment> newSegs) {
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
index 62b46a9d22..33a32514a1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
@@ -37,6 +37,8 @@
private Map<Long, Long> cuboids = null;
private Set<Long> cuboidsRecommend = null;
private Map<String, String> updateTableSnapshotPath = null;
+ private long createTimeUTC = -1;
+ private long cuboidLastOptimized = -1;
public CubeUpdate(CubeInstance cubeInstance) {
setCubeInstance(cubeInstance);
@@ -49,7 +51,7 @@ public CubeInstance getCubeInstance() {
public CubeUpdate setCubeInstance(CubeInstance cubeInstance) {
if (cubeInstance.isCachedAndShared())
throw new IllegalArgumentException();
-
+
this.cubeInstance = cubeInstance;
return this;
}
@@ -133,4 +135,20 @@ public CubeUpdate setCuboidsRecommend(Set<Long> cuboidsRecommend) {
public void setUpdateTableSnapshotPath(Map<String, String> updateTableSnapshotPath) {
this.updateTableSnapshotPath = updateTableSnapshotPath;
}
+
+ public long getCreateTimeUTC() {
+ return createTimeUTC;
+ }
+
+ public void setCreateTimeUTC(long createTimeUTC) {
+ this.createTimeUTC = createTimeUTC;
+ }
+
+ public long getCuboidLastOptimized() {
+ return cuboidLastOptimized;
+ }
+
+ public void setCuboidLastOptimized(long cuboidLastOptimized) {
+ this.cuboidLastOptimized = cuboidLastOptimized;
+ }
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 96d60c71ff..e541c37f14 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -53,13 +53,13 @@
import org.apache.kylin.measure.percentile.PercentileMeasureType;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.draft.Draft;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IStorageAware;
+import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
@@ -95,7 +95,6 @@
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
/**
* Stateless & lightweight service facade of cube management functions.
@@ -580,12 +579,7 @@ protected void releaseAllJobs(CubeInstance cube) {
private void releaseAllSegments(CubeInstance cube) throws IOException {
releaseAllJobs(cube);
- CubeUpdate update = new CubeUpdate(cube.latestCopyForWrite());
- update.setToRemoveSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()]));
- update.setCuboids(Maps.<Long, Long> newHashMap());
- update.setCuboidsRecommend(Sets.<Long> newHashSet());
- update.setUpdateTableSnapshotPath(Maps.<String, String> newHashMap());
- CubeManager.getInstance(getConfig()).updateCube(update);
+ getCubeManager().clearSegments(cube);
}
public void updateOnNewSegmentReady(String cubeName) {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services