You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/06/02 10:46:50 UTC
incubator-kylin git commit: bug fix: rebuild proj L2 cache at cube/II
update
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8.0 36af047a7 -> 4e0d4c318
bug fix: rebuild proj L2 cache at cube/II update
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4e0d4c31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4e0d4c31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4e0d4c31
Branch: refs/heads/0.8.0
Commit: 4e0d4c3181b5e6cf5e025d2f135b7599a29e0c6c
Parents: 36af047
Author: honma <ho...@ebay.com>
Authored: Tue Jun 2 16:46:40 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Jun 2 16:46:40 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 44 +++++++-------------
.../apache/kylin/cube/CubeManagerCacheTest.java | 2 +-
.../apache/kylin/invertedindex/IIManager.java | 12 +++---
.../kylin/job/cube/MergeDictionaryStep.java | 2 +-
.../job/cube/UpdateCubeInfoAfterBuildStep.java | 4 +-
.../job/cube/UpdateCubeInfoAfterMergeStep.java | 3 +-
.../job/hadoop/invertedindex/IIBulkLoadJob.java | 2 +-
.../kylin/job/streaming/CubeStreamBuilder.java | 3 +-
.../kylin/job/BuildCubeWithEngineTest.java | 2 +-
.../apache/kylin/job/BuildIIWithEngineTest.java | 8 ++--
.../apache/kylin/job/BuildIIWithStreamTest.java | 8 ++--
.../java/org/apache/kylin/job/DeployUtil.java | 2 +-
.../job/hadoop/cube/MergeCuboidMapperTest.java | 3 +-
.../job/streaming/CubeStreamBuilderTest.java | 2 +-
.../kylin/rest/controller/CubeController.java | 2 +-
.../apache/kylin/rest/service/CacheService.java | 3 ++
.../apache/kylin/rest/service/CubeService.java | 8 ++--
.../apache/kylin/rest/service/JobService.java | 2 +-
.../kylin/rest/service/CacheServiceTest.java | 6 +--
19 files changed, 51 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 585e503..44153e9 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -153,7 +153,7 @@ public class CubeManager implements IRealizationProvider {
DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, factColumnsPath);
cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
- updateCube(cubeSeg.getCubeInstance(), null, null, Lists.newArrayList(cubeSeg), null, false);
+ updateCube(cubeSeg.getCubeInstance(), null, null, Lists.newArrayList(cubeSeg), null);
return dictInfo;
}
@@ -191,7 +191,7 @@ public class CubeManager implements IRealizationProvider {
cubeSeg.putSnapshotResPath(lookupTable, snapshot.getResourcePath());
- updateCube(cubeSeg.getCubeInstance(), null, null, Lists.newArrayList(cubeSeg), null, false);
+ updateCube(cubeSeg.getCubeInstance(), null, null, Lists.newArrayList(cubeSeg), null);
return snapshot;
}
@@ -226,28 +226,17 @@ public class CubeManager implements IRealizationProvider {
CubeInstance cube = CubeInstance.create(cubeName, projectName, desc);
cube.setOwner(owner);
- updateCube(cube, null, null, null, null, false);
+ updateCube(cube, null, null, null, null);
ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName, owner);
return cube;
}
- /**
- *
- * @param cube
- * @param updateProject Updating project is necessary when you want the project's JDBC connection
- * reflect the content in the cube. So basically you only set it to true when
- * a cube first turns into READY, or when the cube's status changed from READY
- * to un ready.
- * if not sure whether to enable updateProject, just use it
- * @return
- * @throws IOException
- */
- public CubeInstance updateCube(CubeInstance cube, final List<CubeSegment> toAddSegs, final List<CubeSegment> toRemoveSegs, final List<CubeSegment> toUpdateSegs, RealizationStatusEnum newStatus, boolean updateProject) throws IOException {
- return updateCube(cube, toAddSegs, toRemoveSegs, toUpdateSegs, newStatus, updateProject, 0);
+ public CubeInstance updateCube(CubeInstance cube, final List<CubeSegment> toAddSegs, final List<CubeSegment> toRemoveSegs, final List<CubeSegment> toUpdateSegs, RealizationStatusEnum newStatus) throws IOException {
+ return updateCube(cube, toAddSegs, toRemoveSegs, toUpdateSegs, newStatus, 0);
}
- private CubeInstance updateCube(CubeInstance cube, final List<CubeSegment> toAddSegs, final List<CubeSegment> toRemoveSegs, final List<CubeSegment> toUpdateSegs, RealizationStatusEnum newStatus, boolean updateProject, int retry) throws IOException {
+ private CubeInstance updateCube(CubeInstance cube, final List<CubeSegment> toAddSegs, final List<CubeSegment> toRemoveSegs, final List<CubeSegment> toUpdateSegs, RealizationStatusEnum newStatus, int retry) throws IOException {
if (cube == null)
throw new IllegalStateException();
@@ -287,15 +276,13 @@ public class CubeManager implements IRealizationProvider {
}
retry++;
- cube = updateCube(reloadCubeLocal(cube.getName()), toAddSegs, toRemoveSegs, toUpdateSegs, newStatus, updateProject, retry);
+ cube = updateCube(reloadCubeLocal(cube.getName()), toAddSegs, toRemoveSegs, toUpdateSegs, newStatus, retry);
}
cubeMap.put(cube.getName(), cube);
- if (updateProject) {
- logger.info("Updating project instance for cube:'" + cube.getName());
- ProjectManager.getInstance(config).updateProject(RealizationType.CUBE, cube.getName());
- }
+ //this is a duplicate call to take care of scenarios where REST cache service unavailable
+ ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).clearL2Cache();
return cube;
}
@@ -314,12 +301,11 @@ public class CubeManager implements IRealizationProvider {
CubeSegment mergeSegment = newSegment(cube, startDate, endDate);
validateNewSegments(cube, mergeSegment);
- updateCube(cube, Lists.newArrayList(appendSegment, mergeSegment), null, null, null, false);
+ updateCube(cube, Lists.newArrayList(appendSegment, mergeSegment), null, null, null);
return new Pair<CubeSegment, CubeSegment>(appendSegment, mergeSegment);
}
-
public CubeSegment appendSegments(CubeInstance cube, long endDate) throws IOException {
return appendSegments(cube, endDate, true, true);
}
@@ -339,7 +325,7 @@ public class CubeManager implements IRealizationProvider {
validateNewSegments(cube, newSegment);
if (saveChange)
- updateCube(cube, Lists.newArrayList(newSegment), null, null, null, false);
+ updateCube(cube, Lists.newArrayList(newSegment), null, null, null);
return newSegment;
}
@@ -348,7 +334,7 @@ public class CubeManager implements IRealizationProvider {
checkNoBuildingSegment(cube);
CubeSegment newSegment = newSegment(cube, startDate, endDate);
- updateCube(cube, Lists.newArrayList(newSegment), null, null, null, false);
+ updateCube(cube, Lists.newArrayList(newSegment), null, null, null);
return newSegment;
}
@@ -361,7 +347,7 @@ public class CubeManager implements IRealizationProvider {
CubeSegment newSegment = newSegment(cube, range.getFirst(), range.getSecond());
validateNewSegments(cube, newSegment);
- updateCube(cube, Lists.newArrayList(newSegment), null, null, null, false);
+ updateCube(cube, Lists.newArrayList(newSegment), null, null, null);
return newSegment;
}
@@ -549,7 +535,7 @@ public class CubeManager implements IRealizationProvider {
return null;
}
- public void promoteNewlyBuiltSegments(CubeInstance cube, boolean updateProj, CubeSegment... newSegments) throws IOException {
+ public void promoteNewlyBuiltSegments(CubeInstance cube, CubeSegment... newSegments) throws IOException {
List<CubeSegment> tobe = calculateToBeSegments(cube);
for (CubeSegment seg : newSegments) {
@@ -577,7 +563,7 @@ public class CubeManager implements IRealizationProvider {
}
logger.info("Promoting cube " + cube + ", new segments " + newSegments);
- updateCube(cube, Lists.newArrayList(newSegments), toRemoveSegs, null, RealizationStatusEnum.READY, updateProj);
+ updateCube(cube, Lists.newArrayList(newSegments), toRemoveSegs, null, RealizationStatusEnum.READY);
}
public void validateNewSegments(CubeInstance cube, CubeSegment... newSegments) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java b/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
index 6b96bd4..758d045 100644
--- a/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
+++ b/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
@@ -67,7 +67,7 @@ public class CubeManagerCacheTest extends LocalFileMetadataTestCase {
assertEquals(RealizationStatusEnum.DISABLED, createdCube.getStatus());
createdCube.setStatus(RealizationStatusEnum.DESCBROKEN);
- cubeManager.updateCube(createdCube, null, null, null, null, true);
+ cubeManager.updateCube(createdCube, null, null, null, null);
assertEquals(RealizationStatusEnum.DESCBROKEN, cubeManager.getCube("a_whole_new_cube").getStatus());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
index ea40ebd..f478caf 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
@@ -143,7 +143,7 @@ public class IIManager implements IRealizationProvider {
DictionaryInfo dict = dictMgr.buildDictionary(iiDesc.getModel(), "true", column, factColumnsPath);
iiSeg.putDictResPath(column, dict.getResourcePath());
}
- updateII(iiSeg.getIIInstance(), false);
+ updateII(iiSeg.getIIInstance());
}
/**
@@ -174,7 +174,7 @@ public class IIManager implements IRealizationProvider {
if (this.getII(ii.getName()) != null)
throw new IllegalArgumentException("The II name '" + ii.getName() + "' already exists.");
- this.updateII(ii, false);
+ this.updateII(ii);
String projectName = (null == ii.getProjectName()) ? ProjectInstance.DEFAULT_PROJECT_NAME : ii.getProjectName();
ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.INVERTED_INDEX, ii.getName(), projectName, ii.getOwner());
@@ -214,15 +214,13 @@ public class IIManager implements IRealizationProvider {
usedStorageLocation.removeAll(name.toUpperCase());
}
- public void updateII(IIInstance ii, boolean updateProject) throws IOException {
+ public void updateII(IIInstance ii) throws IOException {
logger.info("Updating II instance : " + ii.getName());
getStore().putResource(ii.getResourcePath(), ii, II_SERIALIZER);
iiMap.put(ii.getName(), ii);
- if (updateProject) {
- logger.info("Updating project instance for ii: " + ii.getName());
- ProjectManager.getInstance(config).updateProject(RealizationType.INVERTED_INDEX, ii.getName());
- }
+ //this is a duplicate call to take care of scenarios where REST cache service unavailable
+ ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).clearL2Cache();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java b/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
index 82a8984..bf0308f 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
@@ -63,7 +63,7 @@ public class MergeDictionaryStep extends AbstractExecutable {
makeDictForNewSegment(conf, cube, newSegment, mergingSegments);
makeSnapshotForNewSegment(cube, newSegment, mergingSegments);
- mgr.updateCube(cube, null, null, Lists.newArrayList(newSegment), null, false);
+ mgr.updateCube(cube, null, null, Lists.newArrayList(newSegment), null);
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
} catch (IOException e) {
logger.error("fail to merge dictionary or lookup snapshots", e);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
index e00b36d..bcf7f7c 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
@@ -126,9 +126,9 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
try {
if (segmentReady) {
- cubeManager.promoteNewlyBuiltSegments(cube, cube.getSegments().size() == 1, segment);
+ cubeManager.promoteNewlyBuiltSegments(cube, segment);
} else {
- cubeManager.updateCube(cube, null, null, Lists.newArrayList(segment), null, false);
+ cubeManager.updateCube(cube, null, null, Lists.newArrayList(segment), null);
}
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
index d7b07ac..d50bd6a 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterMergeStep.java
@@ -30,7 +30,6 @@ import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
import java.io.IOException;
import java.util.ArrayList;
@@ -86,7 +85,7 @@ public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
mergedSegment.setLastBuildTime(System.currentTimeMillis());
try {
- cubeManager.promoteNewlyBuiltSegments(cube, false, mergedSegment);
+ cubeManager.promoteNewlyBuiltSegments(cube, mergedSegment);
return new ExecuteResult(ExecuteResult.State.SUCCEED);
} catch (IOException e) {
logger.error("fail to update cube after merge", e);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
index 145a3c8..f636af4 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
@@ -64,7 +64,7 @@ public class IIBulkLoadJob extends AbstractHadoopJob {
IISegment seg = ii.getFirstSegment();
seg.setStorageLocationIdentifier(tableName);
seg.setStatus(SegmentStatusEnum.READY);
- mgr.updateII(ii,true);
+ mgr.updateII(ii);
return hbaseExitCode;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
index 34d4eaf..c62edfe 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
@@ -8,7 +8,6 @@ import com.google.common.collect.Maps;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -341,7 +340,7 @@ public class CubeStreamBuilder extends StreamBuilder {
CubeInstance cube = CubeManager.getInstance(kylinConfig).reloadCubeLocal(cubeSegment.getCubeInstance().getName());
cube.getSegments().add(cubeSegment);
Collections.sort(cube.getSegments());
- CubeManager.getInstance(kylinConfig).updateCube(cube, Lists.newArrayList(cubeSegment), null, null, null, false);
+ CubeManager.getInstance(kylinConfig).updateCube(cube, Lists.newArrayList(cubeSegment), null, null, null);
}
private List<Long> getAllCuboidIds(CubeDesc cubeDesc) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 0888994..986a8b4 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -259,7 +259,7 @@ public class BuildCubeWithEngineTest {
private void clearSegment(String cubeName) throws Exception {
CubeInstance cube = cubeManager.getCube(cubeName);
// remove all existing segments
- cubeManager.updateCube(cube, null, cube.getSegments(), null, null, true);
+ cubeManager.updateCube(cube, null, cube.getSegments(), null, null);
}
private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 1094caa..02bb6b8 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -116,7 +116,7 @@ public class BuildIIWithEngineTest {
IIInstance ii = iiManager.getII(iiInstance);
if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii, true);
+ iiManager.updateII(ii);
}
}
}
@@ -128,7 +128,7 @@ public class BuildIIWithEngineTest {
IIInstance ii = iiManager.getII(iiInstance);
if (ii.getStatus() != RealizationStatusEnum.READY) {
ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii, true);
+ iiManager.updateII(ii);
}
}
backup();
@@ -208,14 +208,14 @@ public class BuildIIWithEngineTest {
private void clearSegment(String iiName) throws Exception {
IIInstance ii = iiManager.getII(iiName);
ii.getSegments().clear();
- iiManager.updateII(ii,true);
+ iiManager.updateII(ii);
}
private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
IIInstance iiInstance = iiManager.getII(iiName);
IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance, true);
+ iiManager.updateII(iiInstance);
IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
IIJob job = iiJobBuilder.buildJob(segment);
jobService.addJob(job);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index 9876ace..a73c5b9 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -108,7 +108,7 @@ public class BuildIIWithStreamTest {
IIInstance ii = iiManager.getII(iiInstance);
if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii,true);
+ iiManager.updateII(ii);
}
}
}
@@ -168,7 +168,7 @@ public class BuildIIWithStreamTest {
private void clearSegment(String iiName) throws Exception {
IIInstance ii = iiManager.getII(iiName);
ii.getSegments().clear();
- iiManager.updateII(ii,true);
+ iiManager.updateII(ii);
}
private IISegment createSegment(String iiName) throws Exception {
@@ -185,7 +185,7 @@ public class BuildIIWithStreamTest {
IIInstance iiInstance = iiManager.getII(iiName);
IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance,true);
+ iiManager.updateII(iiInstance);
return segment;
}
@@ -242,7 +242,7 @@ public class BuildIIWithStreamTest {
IIInstance ii = iiManager.getII(iiName);
if (ii.getStatus() != RealizationStatusEnum.READY) {
ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii,true);
+ iiManager.updateII(ii);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/job/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DeployUtil.java b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
index e238e12..e59ffe4 100644
--- a/job/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -60,7 +60,7 @@ public class DeployUtil {
// update cube desc signature.
for (CubeInstance cube : CubeManager.getInstance(config()).listAllCubes()) {
cube.getDescriptor().setSignature(cube.getDescriptor().calculateSignature());
- CubeManager.getInstance(config()).updateCube(cube, null, null, null, null, true);
+ CubeManager.getInstance(config()).updateCube(cube, null, null, null, null);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
index e9cdb96..d9d6d7c 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
@@ -18,7 +18,6 @@
package org.apache.kylin.job.hadoop.cube;
-import com.google.common.collect.Lists;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
@@ -140,7 +139,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
}
- cube = cubeManager.updateCube(cube, null, null, cube.getSegments(), null, true);
+ cube = cubeManager.updateCube(cube, null, null, cube.getSegments(), null);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/job/src/test/java/org/apache/kylin/job/streaming/CubeStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/streaming/CubeStreamBuilderTest.java b/job/src/test/java/org/apache/kylin/job/streaming/CubeStreamBuilderTest.java
index 96b7cb4..d8d259f 100644
--- a/job/src/test/java/org/apache/kylin/job/streaming/CubeStreamBuilderTest.java
+++ b/job/src/test/java/org/apache/kylin/job/streaming/CubeStreamBuilderTest.java
@@ -51,7 +51,7 @@ public class CubeStreamBuilderTest {
DeployUtil.overrideJobJarLocations();
final CubeInstance cube = CubeManager.getInstance(kylinConfig).getCube(CUBE_NAME);
// remove all existing segments
- CubeManager.getInstance(kylinConfig).updateCube(cube, null, cube.getSegments(), null, null, true);
+ CubeManager.getInstance(kylinConfig).updateCube(cube, null, cube.getSegments(), null, null);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 33a2886..30afb87 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -421,7 +421,7 @@ public class CubeController extends BasicController {
cubeService.getCubeManager().validateNewSegments(cube, segment);
try {
- cubeService.getCubeManager().updateCube(cube, Lists.newArrayList(segment), null, null, null, true);
+ cubeService.getCubeManager().updateCube(cube, Lists.newArrayList(segment), null, null, null);
} catch (IOException e) {
logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index da710d4..d3480ce 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -67,6 +67,8 @@ public class CacheService extends BasicService {
switch (cacheType) {
case CUBE:
CubeInstance newCube = getCubeManager().reloadCubeLocal(cacheKey);
+ getProjectManager().clearL2Cache();
+
//clean query related cache first
super.cleanDataCache(newCube.getUuid());
//move this logic to other place
@@ -82,6 +84,7 @@ public class CacheService extends BasicService {
case INVERTED_INDEX:
//II update does not need to update storage cache because it is dynamic already
getIIManager().reloadIILocal(cacheKey);
+ getProjectManager().clearL2Cache();
break;
case INVERTED_INDEX_DESC:
getIIDescManager().reloadIIDescLocal(cacheKey);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 7e38d73..4c1aaf9 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -150,7 +150,7 @@ public class CubeService extends BasicService {
String owner = SecurityContextHolder.getContext().getAuthentication().getName();
cube.setOwner(owner);
- return getCubeManager().updateCube(cube, null, null, null, null, true);
+ return getCubeManager().updateCube(cube, null, null, null, null);
}
public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeDesc desc) throws IOException {
@@ -348,7 +348,7 @@ public class CubeService extends BasicService {
cube.setStatus(RealizationStatusEnum.DISABLED);
try {
- return getCubeManager().updateCube(cube, null, null, null, RealizationStatusEnum.DISABLED, true);
+ return getCubeManager().updateCube(cube, null, null, null, RealizationStatusEnum.DISABLED);
} catch (IOException e) {
cube.setStatus(ostatus);
throw e;
@@ -384,7 +384,7 @@ public class CubeService extends BasicService {
}
try {
- return getCubeManager().updateCube(cube, null, null, null, RealizationStatusEnum.READY, true);
+ return getCubeManager().updateCube(cube, null, null, null, RealizationStatusEnum.READY);
} catch (IOException e) {
cube.setStatus(ostatus);
throw e;
@@ -527,7 +527,7 @@ public class CubeService extends BasicService {
getExecutableManager().discardJob(cubingJob.getId());
}
}
- return CubeManager.getInstance(getConfig()).updateCube(cube, null, null, cube.getSegments(), null, true);
+ return CubeManager.getInstance(getConfig()).updateCube(cube, null, null, cube.getSegments(), null);
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/server/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
index 3b3cbec..e058499 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -264,7 +264,7 @@ public class JobService extends BasicService {
final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
if (segment != null && segment.getStatus() == SegmentStatusEnum.NEW) {
// Remove all existing segments
- getCubeManager().updateCube(cubeInstance, null, cubeInstance.getSegments(), null, null, false);
+ getCubeManager().updateCube(cubeInstance, null, cubeInstance.getSegments(), null, null);
}
getExecutableManager().discardJob(jobId);
return jobInstance;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e0d4c31/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index e26f387..f0d403b 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -229,10 +229,10 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertEquals(0, cubeManagerB.getCube(cubeName).getSegments().size());
CubeSegment segment = new CubeSegment();
segment.setName("test_segment");
- cube = cubeManager.updateCube(cube, Lists.newArrayList(segment), null, null, null, true);
+ cube = cubeManager.updateCube(cube, Lists.newArrayList(segment), null, null, null);
//one for cube update, one for project update
- assertEquals(2, broadcaster.getCounterAndClear());
- waitForCounterAndClear(2);
+ assertEquals(1, broadcaster.getCounterAndClear());
+ waitForCounterAndClear(1);
assertEquals(1, cubeManagerB.getCube(cubeName).getSegments().size());
assertEquals(segment.getName(), cubeManagerB.getCube(cubeName).getSegments().get(0).getName());