You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/11/12 07:46:18 UTC
[2/2] incubator-kylin git commit: KYLIN-1127 Drop
getProject()/setProject() in IRealization. Pass test on CacheService.
KYLIN-1127 Drop getProject()/setProject() in IRealization. Pass test on CacheService.
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/3ad57284
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/3ad57284
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/3ad57284
Branch: refs/heads/2.x-staging
Commit: 3ad5728497f695b28dbea9f232d80d1e06e51d0f
Parents: 86a2b6e
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Nov 12 14:45:37 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Nov 12 14:45:37 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeDescManager.java | 2 +-
.../org/apache/kylin/cube/CubeInstance.java | 13 +--------
.../java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../kylin/metadata/project/ProjectManager.java | 20 --------------
.../metadata/realization/IRealization.java | 4 ---
.../kylin/storage/hybrid/HybridInstance.java | 11 --------
.../apache/kylin/invertedindex/IIInstance.java | 10 -------
.../apache/kylin/invertedindex/IIManager.java | 3 +-
.../apache/kylin/rest/service/CacheService.java | 29 ++++++++++++++++----
.../apache/kylin/rest/service/CubeService.java | 1 -
10 files changed, 28 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index c50836c..53cd00f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -229,7 +229,7 @@ public class CubeDescManager {
* @return
* @throws IOException
*/
- public CubeDesc updateCubeDesc(CubeDesc desc) throws IOException {
+ public CubeDesc updateCubeDesc(CubeDesc desc) throws IOException {
// Validate CubeDesc
if (desc.getUuid() == null || desc.getName() == null) {
throw new IllegalArgumentException();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index f8e37f3..2208136 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -52,7 +52,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
private static final int COST_WEIGHT_DIMENSION = 10;
private static final int COST_WEIGHT_INNER_JOIN = 100;
- public static CubeInstance create(String cubeName, String projectName, CubeDesc cubeDesc) {
+ public static CubeInstance create(String cubeName, CubeDesc cubeDesc) {
CubeInstance cubeInstance = new CubeInstance();
cubeInstance.setConfig(cubeDesc.getConfig());
@@ -62,7 +62,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
cubeInstance.setSegments(new ArrayList<CubeSegment>());
cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
cubeInstance.updateRandomUuid();
- cubeInstance.setProjectName(projectName);
return cubeInstance;
}
@@ -90,8 +89,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
@JsonProperty("create_time_utc")
private long createTimeUTC;
- private String projectName;
-
// default constructor for jackson
public CubeInstance() {
}
@@ -378,14 +375,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return Lists.newArrayList(getDescriptor().listAllColumns());
}
- public String getProjectName() {
- return projectName;
- }
-
- public void setProjectName(String projectName) {
- this.projectName = projectName;
- }
-
@Override
public long getDateRangeStart() {
List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 89873d2..4592b15 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -248,7 +248,7 @@ public class CubeManager implements IRealizationProvider {
logger.info("Creating cube '" + projectName + "-->" + cubeName + "' from desc '" + desc.getName() + "'");
// save cube resource
- CubeInstance cube = CubeInstance.create(cubeName, projectName, desc);
+ CubeInstance cube = CubeInstance.create(cubeName, desc);
cube.setOwner(owner);
updateCubeWithRetry(new CubeUpdate(cube), 0);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index 8304128..37241c3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -20,7 +20,6 @@ package org.apache.kylin.metadata.project;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -36,7 +35,6 @@ import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.kylin.metadata.realization.RealizationType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -105,7 +103,6 @@ public class ProjectManager {
for (String path : paths) {
reloadProjectLocalAt(path);
}
- wireProjectAndRealizations(projectMap.values());
logger.debug("Loaded " + projectMap.size() + " Project(s)");
}
@@ -129,23 +126,6 @@ public class ProjectManager {
return projectInstance;
}
- private void wireProjectAndRealizations(Collection<ProjectInstance> projectInstances) {
- if (projectInstances.isEmpty())
- return;
-
- RealizationRegistry registry = RealizationRegistry.getInstance(config);
- for (ProjectInstance projectInstance : projectInstances) {
- for (RealizationEntry realization : projectInstance.getRealizationEntries()) {
- IRealization rel = registry.getRealization(realization.getType(), realization.getRealization());
- if (rel != null) {
- rel.setProjectName(projectInstance.getName());
- } else {
- logger.warn("Realization '" + realization + "' defined under project '" + projectInstance + "' is not found");
- }
- }
- }
- }
-
public List<ProjectInstance> listAllProjects() {
return new ArrayList<ProjectInstance>(projectMap.values());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index 8c9258a..ccfa956 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -65,10 +65,6 @@ public interface IRealization extends IStorageAware {
public String getCanonicalName();
- public String getProjectName();
-
- public void setProjectName(String prjName);
-
public long getDateRangeStart();
public long getDateRangeEnd();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 4049fcd..2a6e5d3 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -66,7 +66,6 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
private long dateRangeStart;
private long dateRangeEnd;
private boolean isReady = false;
- private String projectName;
private boolean initiated = false;
private final static Logger logger = LoggerFactory.getLogger(HybridInstance.class);
@@ -226,16 +225,6 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
return getType() + "[name=" + name + "]";
}
- @Override
- public String getProjectName() {
- return projectName;
- }
-
- @Override
- public void setProjectName(String prjName) {
- projectName = prjName;
- }
-
public KylinConfig getConfig() {
return config;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
index 20f289c..0ba07ee 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
@@ -81,8 +81,6 @@ public class IIInstance extends RootPersistentEntity implements IRealization, IB
@JsonProperty("create_time_utc")
private long createTimeUTC;
- private String projectName;
-
private static final int COST_WEIGHT_LOOKUP_TABLE = 1;
private static final int COST_WEIGHT_INNER_JOIN = 2;
@@ -314,14 +312,6 @@ public class IIInstance extends RootPersistentEntity implements IRealization, IB
return getDescriptor().getMeasures();
}
- public String getProjectName() {
- return projectName;
- }
-
- public void setProjectName(String projectName) {
- this.projectName = projectName;
- }
-
@Override
public long getDateRangeStart() {
List<IISegment> readySegs = getSegments(SegmentStatusEnum.READY);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
index b6dfdf1..06aeb04 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
@@ -134,7 +134,8 @@ public class IIManager implements IRealizationProvider {
this.updateII(ii);
- String projectName = (null == ii.getProjectName()) ? ProjectInstance.DEFAULT_PROJECT_NAME : ii.getProjectName();
+ // FIXME need to pass in project name
+ String projectName = ProjectInstance.DEFAULT_PROJECT_NAME;
ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.INVERTED_INDEX, ii.getName(), projectName, ii.getOwner());
return ii;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 8371907..0150b2e 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -118,7 +118,7 @@ public class CacheService extends BasicService {
}
}
- protected void removeOLAPDataSource(String project) {
+ private static void removeOLAPDataSource(String project) {
logger.info("removeOLAPDataSource is called for project " + project);
if (StringUtils.isEmpty(project))
throw new IllegalArgumentException("removeOLAPDataSource: project name not given");
@@ -186,8 +186,7 @@ public class CacheService extends BasicService {
getCubeDescManager().reloadCubeDescLocal(cacheKey);
break;
case PROJECT:
- getProjectManager().reloadProjectLocal(cacheKey);
- removeOLAPDataSource(cacheKey);
+ reloadProjectCache(cacheKey);
break;
case INVERTED_INDEX:
//II update does not need to update storage cache because it is dynamic already
@@ -234,7 +233,7 @@ public class CacheService extends BasicService {
private void rebuildCubeCache(String cubeName) {
CubeInstance cube = getCubeManager().reloadCubeLocal(cubeName);
getHybridManager().reloadHybridInstanceByChild(RealizationType.CUBE, cubeName);
- getProjectManager().clearL2Cache();
+ reloadProjectCache(getProjectManager().findProjects(RealizationType.CUBE, cubeName));
//clean query related cache first
if (cube != null) {
cleanDataCache(cube.getUuid());
@@ -278,11 +277,29 @@ public class CacheService extends BasicService {
if (cube == null) {
cube = getCubeManager().getCube(cubeName);
}
-
+
getCubeManager().removeCubeLocal(cubeName);
-
+ getHybridManager().reloadHybridInstanceByChild(RealizationType.CUBE, cubeName);
+ reloadProjectCache(getProjectManager().findProjects(RealizationType.CUBE, cubeName));
+
if (cube != null) {
cleanDataCache(cube.getUuid());
}
}
+
+ private void reloadProjectCache(List<ProjectInstance> projects) {
+ for (ProjectInstance prj : projects) {
+ reloadProjectCache(prj.getName());
+ }
+ }
+
+ private void reloadProjectCache(String projectName) {
+ try {
+ getProjectManager().reloadProjectLocal(projectName);
+ } catch (IOException ex) {
+ logger.warn("Failed to reset project cache", ex);
+ }
+ removeOLAPDataSource(projectName);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index bcbd9f8..e9d446f 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -397,7 +397,6 @@ public class CubeService extends BasicService {
}
try {
-
CubeUpdate cubeBuilder = new CubeUpdate(cube);
cubeBuilder.setStatus(RealizationStatusEnum.READY);
return getCubeManager().updateCube(cubeBuilder);