You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/05/26 17:20:50 UTC
[04/32] incubator-kylin git commit: fix ci: clean broadcase module
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index d168ddd..ba62bb2 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -63,7 +63,6 @@ import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
import java.io.IOException;
-import java.net.UnknownHostException;
import java.util.*;
/**
@@ -81,7 +80,7 @@ public class CubeService extends BasicService {
private AccessService accessService;
@PostFilter(Constant.ACCESS_POST_FILTER_READ)
- public List<CubeInstance> listAllCubes(final String cubeName, final String projectName,final String modelName) {
+ public List<CubeInstance> listAllCubes(final String cubeName, final String projectName, final String modelName) {
List<CubeInstance> cubeInstances = null;
ProjectInstance project = (null != projectName) ? getProjectManager().getProject(projectName) : null;
@@ -93,14 +92,14 @@ public class CubeService extends BasicService {
List<CubeInstance> filterModelCubes = new ArrayList<CubeInstance>();
- if(modelName!=null){
+ if (modelName != null) {
for (CubeInstance cubeInstance : cubeInstances) {
boolean isCubeMatch = cubeInstance.getDescriptor().getModelName().toLowerCase().equals(modelName.toLowerCase());
if (isCubeMatch) {
filterModelCubes.add(cubeInstance);
}
}
- }else{
+ } else {
filterModelCubes = cubeInstances;
}
@@ -116,12 +115,12 @@ public class CubeService extends BasicService {
return filterCubes;
}
- public List<CubeInstance> getCubes(final String cubeName, final String projectName,final String modelName ,final Integer limit, final Integer offset) {
+ public List<CubeInstance> getCubes(final String cubeName, final String projectName, final String modelName, final Integer limit, final Integer offset) {
List<CubeInstance> cubes;
- cubes = listAllCubes(cubeName, projectName,modelName);
+ cubes = listAllCubes(cubeName, projectName, modelName);
- if(limit==null||offset==null){
+ if (limit == null || offset == null) {
return cubes;
}
@@ -154,7 +153,7 @@ public class CubeService extends BasicService {
String owner = SecurityContextHolder.getContext().getAuthentication().getName();
cube.setOwner(owner);
- return getCubeManager().updateCube(cube);
+ return getCubeManager().updateCube(cube, true);
}
public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeDesc desc) throws IOException {
@@ -174,7 +173,6 @@ public class CubeService extends BasicService {
createdDesc = getCubeDescManager().updateCubeDesc(desc);
}
-
if (!createdDesc.getError().isEmpty()) {
if (isNew) {
getCubeDescManager().removeCubeDesc(createdDesc);
@@ -235,7 +233,7 @@ public class CubeService extends BasicService {
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
- public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName) throws IOException, JobException {
+ public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName) throws IOException, JobException {
final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
if (!cubingJobs.isEmpty()) {
throw new JobException("Cube schema shouldn't be changed with running job.");
@@ -276,20 +274,14 @@ public class CubeService extends BasicService {
accessService.clean(cube, true);
}
- public boolean isCubeEditable(CubeInstance ci) {
- return ci.getStatus() == RealizationStatusEnum.DISABLED;
- }
-
public boolean isCubeDescEditable(CubeDesc cd) {
- List<CubeInstance> list = getCubeManager().getCubesByDesc(cd.getName());
- if (list.isEmpty()) {
+ String cubeName = getCubeNameFromDesc(cd.getName());
+ CubeInstance cube = getCubeManager().getCube(cubeName);
+ if (cube == null) {
return true;
}
- Iterator<CubeInstance> it = list.iterator();
- while (it.hasNext()) {
- if (!isCubeEditable(it.next())) {
- return false;
- }
+ if (cube.getSegments().size() != 0) {
+ return false;
}
return true;
}
@@ -306,14 +298,6 @@ public class CubeService extends BasicService {
}
}
- public void reloadCubeCache(String cubeName) {
- CubeManager.getInstance(this.getConfig()).loadCubeCache(cubeName);
- }
-
- public void removeCubeCache(String cubeName) {
- CubeManager.getInstance(this.getConfig()).removeCubeCacheLocal(cubeName);
- }
-
/**
* Stop all jobs belonging to this cube and clean out all segments
*
@@ -323,7 +307,7 @@ public class CubeService extends BasicService {
* @throws JobException
*/
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
- @Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
+ @Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) })
public CubeInstance purgeCube(CubeInstance cube) throws IOException, JobException {
String cubeName = cube.getName();
@@ -349,7 +333,7 @@ public class CubeService extends BasicService {
* @throws JobException
*/
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
- @Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
+ @Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) })
public CubeInstance disableCube(CubeInstance cube) throws IOException, JobException {
String cubeName = cube.getName();
@@ -361,7 +345,7 @@ public class CubeService extends BasicService {
cube.setStatus(RealizationStatusEnum.DISABLED);
try {
- return getCubeManager().updateCube(cube);
+ return getCubeManager().updateCube(cube, true);
} catch (IOException e) {
cube.setStatus(ostatus);
throw e;
@@ -398,7 +382,7 @@ public class CubeService extends BasicService {
cube.setStatus(RealizationStatusEnum.READY);
try {
- return getCubeManager().updateCube(cube);
+ return getCubeManager().updateCube(cube, true);
} catch (IOException e) {
cube.setStatus(ostatus);
throw e;
@@ -511,7 +495,6 @@ public class CubeService extends BasicService {
getExecutableManager().addJob(job);
}
-
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
public void updateCubeNotifyList(CubeInstance cube, List<String> notifyList) throws IOException {
CubeDesc desc = cube.getDescriptor();
@@ -543,7 +526,7 @@ public class CubeService extends BasicService {
}
}
cube.getSegments().clear();
- CubeManager.getInstance(getConfig()).updateCube(cube);
+ CubeManager.getInstance(getConfig()).updateCube(cube, true);
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
@@ -557,7 +540,6 @@ public class CubeService extends BasicService {
getProjectManager().addTableDescToProject(tables, project);
}
-
@PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
public void calculateCardinalityIfNotPresent(String[] tables, String submitter) throws IOException {
MetadataManager metaMgr = getMetadataManager();
@@ -569,5 +551,4 @@ public class CubeService extends BasicService {
}
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/server/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
index e370a27..7c96801 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -100,20 +100,20 @@ public class JobService extends BasicService {
private ExecutableState parseToExecutableState(JobStatusEnum status) {
switch (status) {
- case DISCARDED:
- return ExecutableState.DISCARDED;
- case ERROR:
- return ExecutableState.ERROR;
- case FINISHED:
- return ExecutableState.SUCCEED;
- case NEW:
- return ExecutableState.READY;
- case PENDING:
- return ExecutableState.READY;
- case RUNNING:
- return ExecutableState.RUNNING;
- default:
- throw new RuntimeException("illegal status:" + status);
+ case DISCARDED:
+ return ExecutableState.DISCARDED;
+ case ERROR:
+ return ExecutableState.ERROR;
+ case FINISHED:
+ return ExecutableState.SUCCEED;
+ case NEW:
+ return ExecutableState.READY;
+ case PENDING:
+ return ExecutableState.READY;
+ case RUNNING:
+ return ExecutableState.RUNNING;
+ default:
+ throw new RuntimeException("illegal status:" + status);
}
}
@@ -213,37 +213,37 @@ public class JobService extends BasicService {
private JobStatusEnum parseToJobStatus(ExecutableState state) {
switch (state) {
- case READY:
- return JobStatusEnum.PENDING;
- case RUNNING:
- return JobStatusEnum.RUNNING;
- case ERROR:
- return JobStatusEnum.ERROR;
- case DISCARDED:
- return JobStatusEnum.DISCARDED;
- case SUCCEED:
- return JobStatusEnum.FINISHED;
- case STOPPED:
- default:
- throw new RuntimeException("invalid state:" + state);
+ case READY:
+ return JobStatusEnum.PENDING;
+ case RUNNING:
+ return JobStatusEnum.RUNNING;
+ case ERROR:
+ return JobStatusEnum.ERROR;
+ case DISCARDED:
+ return JobStatusEnum.DISCARDED;
+ case SUCCEED:
+ return JobStatusEnum.FINISHED;
+ case STOPPED:
+ default:
+ throw new RuntimeException("invalid state:" + state);
}
}
private JobStepStatusEnum parseToJobStepStatus(ExecutableState state) {
switch (state) {
- case READY:
- return JobStepStatusEnum.PENDING;
- case RUNNING:
- return JobStepStatusEnum.RUNNING;
- case ERROR:
- return JobStepStatusEnum.ERROR;
- case DISCARDED:
- return JobStepStatusEnum.DISCARDED;
- case SUCCEED:
- return JobStepStatusEnum.FINISHED;
- case STOPPED:
- default:
- throw new RuntimeException("invalid state:" + state);
+ case READY:
+ return JobStepStatusEnum.PENDING;
+ case RUNNING:
+ return JobStepStatusEnum.RUNNING;
+ case ERROR:
+ return JobStepStatusEnum.ERROR;
+ case DISCARDED:
+ return JobStepStatusEnum.DISCARDED;
+ case SUCCEED:
+ return JobStepStatusEnum.FINISHED;
+ case STOPPED:
+ default:
+ throw new RuntimeException("invalid state:" + state);
}
}
@@ -264,7 +264,7 @@ public class JobService extends BasicService {
final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
if (segment != null && segment.getStatus() == SegmentStatusEnum.NEW) {
cubeInstance.getSegments().remove(segment);
- getCubeManager().updateCube(cubeInstance);
+ getCubeManager().updateCube(cubeInstance, true);
}
getExecutableManager().discardJob(jobId);
return jobInstance;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index 2013a9f..abaecd2 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -18,12 +18,12 @@
package org.apache.kylin.rest.service;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.request.CreateProjectRequest;
import org.apache.kylin.rest.request.UpdateProjectRequest;
+import org.apache.kylin.rest.security.AclPermission;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -31,11 +31,9 @@ import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.rest.request.CreateProjectRequest;
-import org.apache.kylin.rest.security.AclPermission;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
/**
* @author xduo
@@ -107,12 +105,5 @@ public class ProjectService extends BasicService {
accessService.clean(project, true);
}
- public void reloadProjectCache(String name) throws IOException {
- getProjectManager().reloadProject(name);
- }
-
- public void removeProjectCache(String name) {
- ProjectManager.clearCache();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 7a88ba9..559ec89 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -77,7 +77,13 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
context.addServlet(new ServletHolder(new BroadcasterReceiveServlet(new BroadcasterReceiveServlet.BroadcasterHandler() {
@Override
public void handle(String type, String name, String event) {
- final CacheService cacheService = new CacheService() {
+ final CacheService serviceA = new CacheService() {
+ @Override
+ public KylinConfig getConfig() {
+ return configA;
+ }
+ };
+ final CacheService serviceB = new CacheService() {
@Override
public KylinConfig getConfig() {
return configB;
@@ -87,20 +93,26 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
Broadcaster.EVENT wipeEvent = Broadcaster.EVENT.getEvent(event);
final String log = "wipe cache type: " + wipeType + " event:" + wipeEvent + " name:" + name;
logger.info(log);
- counter.incrementAndGet();
- switch (wipeEvent) {
+ try {
+ switch (wipeEvent) {
case CREATE:
case UPDATE:
- cacheService.rebuildCache(wipeType, name);
+ serviceA.rebuildCache(wipeType, name);
+ serviceB.rebuildCache(wipeType, name);
break;
case DROP:
- cacheService.removeCache(wipeType, name);
+ serviceA.removeCache(wipeType, name);
+ serviceB.removeCache(wipeType, name);
break;
default:
throw new RuntimeException("invalid type:" + wipeEvent);
+ }
+ } finally {
+ counter.incrementAndGet();
}
}
})), "/");
+
server.start();
}
@@ -146,12 +158,15 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
private static CubeManager getCubeManager(KylinConfig config) throws Exception {
return CubeManager.getInstance(config);
}
+
private static ProjectManager getProjectManager(KylinConfig config) throws Exception {
return ProjectManager.getInstance(config);
}
+
private static CubeDescManager getCubeDescManager(KylinConfig config) throws Exception {
return CubeDescManager.getInstance(config);
}
+
private static MetadataManager getMetadataManager(KylinConfig config) throws Exception {
return MetadataManager.getInstance(config);
}
@@ -200,10 +215,11 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
cubeManager.createCube(cubeName, ProjectInstance.DEFAULT_PROJECT_NAME, cubeDesc, null);
- assertNotNull(cubeManager.getCube(cubeName));
//one for cube update, one for project update
assertEquals(2, broadcaster.getCounterAndClear());
waitForCounterAndClear(2);
+
+ assertNotNull(cubeManager.getCube(cubeName));
assertNotNull(cubeManagerB.getCube(cubeName));
assertTrue(containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
assertTrue(containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
@@ -215,24 +231,24 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
CubeSegment segment = new CubeSegment();
segment.setName("test_segment");
cube.getSegments().add(segment);
- cubeManager.updateCube(cube);
- //only one for update cube
- assertEquals(1, broadcaster.getCounterAndClear());
- waitForCounterAndClear(1);
+ cubeManager.updateCube(cube, true);
+ //one for cube update, one for project update
+ assertEquals(2, broadcaster.getCounterAndClear());
+ waitForCounterAndClear(2);
assertEquals(1, cubeManagerB.getCube(cubeName).getSegments().size());
assertEquals(segment.getName(), cubeManagerB.getCube(cubeName).getSegments().get(0).getName());
//delete cube
cubeManager.dropCube(cubeName, false);
- assertTrue(cubeManager.getCube(cubeName) == null);
- assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
//one for cube update, one for project update
assertEquals(2, broadcaster.getCounterAndClear());
waitForCounterAndClear(2);
+
+ assertTrue(cubeManager.getCube(cubeName) == null);
+ assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
assertTrue(cubeManagerB.getCube(cubeName) == null);
assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
-
final String cubeDescName = "test_cube_desc";
cubeDesc.setName(cubeDescName);
cubeDesc.setLastModified(0);
@@ -245,7 +261,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertNotNull(cubeDescManager.getCubeDesc(cubeDescName));
assertNotNull(cubeDescManagerB.getCubeDesc(cubeDescName));
-
cubeDesc.setNotifyList(Arrays.asList("test@email", "test@email", "test@email"));
cubeDescManager.updateCubeDesc(cubeDesc);
assertEquals(1, broadcaster.getCounterAndClear());
@@ -259,8 +274,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);
-
-
getStore().deleteResource("/cube/a_whole_new_cube.json");
}
@@ -273,7 +286,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
return tableDesc;
}
-
@Test
public void testMetaCRUD() throws Exception {
final MetadataManager metadataManager = MetadataManager.getInstance(configA);
@@ -291,9 +303,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertNotNull(metadataManager.getTableDesc(tableDesc.getIdentity()));
assertNotNull(metadataManagerB.getTableDesc(tableDesc.getIdentity()));
-
-
-
final String dataModelName = "test_data_model";
DataModelDesc dataModelDesc = metadataManager.getDataModelDesc("test_kylin_left_join_model_desc");
dataModelDesc.setName(dataModelName);
@@ -302,7 +311,7 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertTrue(metadataManagerB.getDataModelDesc(dataModelName) == null);
dataModelDesc.setName(dataModelName);
- metadataManager.createDataModelDesc(dataModelDesc,"default","ADMIN");
+ metadataManager.createDataModelDesc(dataModelDesc, "default", "ADMIN");
//one for data model creation, one for project meta update
assertEquals(2, broadcaster.getCounterAndClear());
waitForCounterAndClear(2);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
index 88e31f4..4cf3952 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
@@ -18,17 +18,16 @@
package org.apache.kylin.rest.service;
-import java.net.UnknownHostException;
-import java.util.List;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-
import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.junit.Assert;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.net.UnknownHostException;
+import java.util.List;
/**
* @author xduo
@@ -52,7 +51,6 @@ public class CubeServiceTest extends ServiceTestBase {
Assert.assertNotNull(cubes);
CubeInstance cube = cubes.get(0);
cubeService.isCubeDescEditable(cube.getDescriptor());
- cubeService.isCubeEditable(cube);
cubes = cubeService.getCubes(null, null,null, 1, 0);
Assert.assertTrue(cubes.size() == 1);