You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/15 13:26:42 UTC
[26/50] [abbrv] incubator-kylin git commit: refactor wipe cache policy
refactor wipe cache policy
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/fe439a3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/fe439a3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/fe439a3d
Branch: refs/heads/inverted-index
Commit: fe439a3dcfa2eb5d0a443eec3e205fe11020ebe1
Parents: 37b21de
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Jan 14 16:16:57 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Jan 14 16:16:57 2015 +0800
----------------------------------------------------------------------
.../common/restclient/AbstractRestCache.java | 5 +-
.../common/restclient/Broadcaster.java | 2 +-
.../common/restclient/SingleValueCache.java | 12 ++--
.../com/kylinolap/cube/CubeDescManager.java | 2 +-
.../java/com/kylinolap/cube/CubeManager.java | 6 +-
.../kylinolap/invertedindex/IIDescManager.java | 2 +-
.../com/kylinolap/invertedindex/IIManager.java | 1 -
.../com/kylinolap/metadata/MetadataManager.java | 75 +++++++-------------
.../rest/controller/CacheController.java | 32 ++++++---
.../kylinolap/rest/service/BasicService.java | 8 ---
.../com/kylinolap/rest/service/CubeService.java | 4 +-
11 files changed, 64 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fe439a3d/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java b/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
index b5cc78e..020ccd9 100644
--- a/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
+++ b/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
@@ -28,9 +28,8 @@ public abstract class AbstractRestCache<K, V> {
this.syncType = syncType;
}
- protected final void syncRemote(Object key, Broadcaster.EVENT syncAction) {
- String syncKey = (syncType == Broadcaster.TYPE.METADATA) ? "metadata" : key.toString();
- Broadcaster.getInstance().queue(syncType.getType(), syncAction.getType(), syncKey);
+ protected final void syncRemote(K key, Broadcaster.EVENT syncAction) {
+ Broadcaster.getInstance().queue(syncType.getType(), syncAction.getType(), key.toString());
}
public abstract void put(K key, V value);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fe439a3d/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java b/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
index 67c94c1..c9f477f 100644
--- a/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
+++ b/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
@@ -131,7 +131,7 @@ public class Broadcaster {
}
public static enum TYPE {
- CUBE("cube"), METADATA("metadata"), PROJECT("project"), INVERTED_INDEX("inverted_index");
+ CUBE("cube"), PROJECT("project"), INVERTED_INDEX("inverted_index"), TABLE("table"), DATA_MODEL("data_model");
private String text;
private TYPE(String text) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fe439a3d/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java b/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
index 394697c..07c6723 100644
--- a/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
+++ b/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
/**
* @author xjiang
@@ -28,7 +29,7 @@ import java.util.concurrent.ConcurrentHashMap;
*/
public class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
- private final Map<K, V> innerCache;
+ private final ConcurrentMap<K, V> innerCache;
public SingleValueCache(Broadcaster.TYPE syncType) {
super(syncType);
@@ -36,9 +37,12 @@ public class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
}
public void put(K key, V value) {
- Broadcaster.EVENT eventType = innerCache.containsKey(key) ? Broadcaster.EVENT.UPDATE : Broadcaster.EVENT.CREATE;
- innerCache.put(key, value);
- syncRemote(key, eventType);
+ final V result = innerCache.put(key, value);
+ if (result == null) {
+ syncRemote(key, Broadcaster.EVENT.CREATE);
+ } else {
+ syncRemote(key, Broadcaster.EVENT.UPDATE);
+ }
}
public void putLocal(K key, V value) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fe439a3d/cube/src/main/java/com/kylinolap/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/CubeDescManager.java b/cube/src/main/java/com/kylinolap/cube/CubeDescManager.java
index 22a1558..fc2afd9 100644
--- a/cube/src/main/java/com/kylinolap/cube/CubeDescManager.java
+++ b/cube/src/main/java/com/kylinolap/cube/CubeDescManager.java
@@ -54,7 +54,7 @@ public class CubeDescManager {
private KylinConfig config;
// name ==> CubeDesc
- private SingleValueCache<String, CubeDesc> cubeDescMap = new SingleValueCache<String, CubeDesc>(Broadcaster.TYPE.METADATA);
+ private SingleValueCache<String, CubeDesc> cubeDescMap = new SingleValueCache<String, CubeDesc>(Broadcaster.TYPE.CUBE);
public static CubeDescManager getInstance(KylinConfig config) {
CubeDescManager r = CACHE.get(config);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fe439a3d/cube/src/main/java/com/kylinolap/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/CubeManager.java b/cube/src/main/java/com/kylinolap/cube/CubeManager.java
index 46b60f5..2db819c 100644
--- a/cube/src/main/java/com/kylinolap/cube/CubeManager.java
+++ b/cube/src/main/java/com/kylinolap/cube/CubeManager.java
@@ -216,7 +216,7 @@ public class CubeManager implements IRealizationProvider {
ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName);
// clean cube cache
- this.afterCubeDroped(cube);
+ this.afterCubeDropped(cube);
return cube;
}
@@ -557,12 +557,10 @@ public class CubeManager implements IRealizationProvider {
}
private void afterCubeUpdated(CubeInstance updatedCube) {
- MetadataManager.getInstance(config).reload();
cubeMap.put(updatedCube.getName().toUpperCase(), updatedCube);
}
- private void afterCubeDroped(CubeInstance droppedCube) {
- MetadataManager.getInstance(config).reload();
+ private void afterCubeDropped(CubeInstance droppedCube) {
removeCubeCache(droppedCube);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fe439a3d/invertedindex/src/main/java/com/kylinolap/invertedindex/IIDescManager.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/com/kylinolap/invertedindex/IIDescManager.java b/invertedindex/src/main/java/com/kylinolap/invertedindex/IIDescManager.java
index d1b4578..bee55d2 100644
--- a/invertedindex/src/main/java/com/kylinolap/invertedindex/IIDescManager.java
+++ b/invertedindex/src/main/java/com/kylinolap/invertedindex/IIDescManager.java
@@ -51,7 +51,7 @@ public class IIDescManager {
private KylinConfig config;
// name ==> IIDesc
- private SingleValueCache<String, IIDesc> iiDescMap = new SingleValueCache<String, IIDesc>(Broadcaster.TYPE.METADATA);
+ private SingleValueCache<String, IIDesc> iiDescMap = new SingleValueCache<String, IIDesc>(Broadcaster.TYPE.INVERTED_INDEX);
public static IIDescManager getInstance(KylinConfig config) {
IIDescManager r = CACHE.get(config);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fe439a3d/invertedindex/src/main/java/com/kylinolap/invertedindex/IIManager.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/com/kylinolap/invertedindex/IIManager.java b/invertedindex/src/main/java/com/kylinolap/invertedindex/IIManager.java
index cb1189e..c961e80 100644
--- a/invertedindex/src/main/java/com/kylinolap/invertedindex/IIManager.java
+++ b/invertedindex/src/main/java/com/kylinolap/invertedindex/IIManager.java
@@ -220,7 +220,6 @@ public class IIManager implements IRealizationProvider {
}
private void afterIIUpdated(IIInstance updatedII) {
- MetadataManager.getInstance(config).reload();
iiMap.put(updatedII.getName().toUpperCase(), updatedII);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fe439a3d/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java b/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java
index 46a1f91..6797328 100644
--- a/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java
+++ b/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java
@@ -90,19 +90,15 @@ public class MetadataManager {
CACHE.remove(config);
}
- public static void dropCache() {
- CACHE.clear();
- }
-
// ============================================================================
private KylinConfig config;
// table name ==> SourceTable
- private SingleValueCache<String, TableDesc> srcTableMap = new SingleValueCache<String, TableDesc>(Broadcaster.TYPE.METADATA);
+ private SingleValueCache<String, TableDesc> srcTableMap = new SingleValueCache<String, TableDesc>(Broadcaster.TYPE.TABLE);
// name => value
- private SingleValueCache<String, Map<String, String>> srcTableExdMap = new SingleValueCache<String, Map<String, String>>(Broadcaster.TYPE.METADATA);
+ private SingleValueCache<String, Map<String, String>> srcTableExdMap = new SingleValueCache<String, Map<String, String>>(Broadcaster.TYPE.TABLE);
// name => DataModelDesc
- private SingleValueCache<String, DataModelDesc> dataModelDescMap = new SingleValueCache<String, DataModelDesc>(Broadcaster.TYPE.METADATA);
+ private SingleValueCache<String, DataModelDesc> dataModelDescMap = new SingleValueCache<String, DataModelDesc>(Broadcaster.TYPE.DATA_MODEL);
private MetadataManager(KylinConfig config) throws IOException {
init(config);
@@ -208,10 +204,6 @@ public class MetadataManager {
logger.debug("Loaded " + srcTableExdMap.size() + " SourceTable EXD(s)");
}
- public Map<String, String> reloadSourceTableExt(String tableIdentity) throws IOException {
- return reloadSourceTableExdAt(TableDesc.concatExdResourcePath(tableIdentity));
- }
-
@SuppressWarnings("unchecked")
private Map<String, String> reloadSourceTableExdAt(String path) throws IOException {
Map<String, String> attrs = Maps.newHashMap();
@@ -235,10 +227,8 @@ public class MetadataManager {
if (file.indexOf("/") > -1) {
file = file.substring(file.lastIndexOf("/") + 1);
}
- String tableIdentity = file.substring(0, file.length() - MetadataConstances.FILE_SURFIX.length());
+ String tableIdentity = file.substring(0, file.length() - MetadataConstances.FILE_SURFIX.length()).toUpperCase();
- checkNoDupName(tableIdentity, srcTableExdMap.containsKey(tableIdentity), "SourceTableExd", path);
-
srcTableExdMap.putLocal(tableIdentity, attrs);
return attrs;
}
@@ -257,22 +247,31 @@ public class MetadataManager {
logger.debug("Loaded " + srcTableMap.size() + " SourceTable(s)");
}
- public TableDesc reloadSourceTable(String tableIdentity) throws IOException {
- return reloadSourceTableAt(TableDesc.concatResourcePath(tableIdentity));
- }
-
private TableDesc reloadSourceTableAt(String path) throws IOException {
ResourceStore store = getStore();
TableDesc t = store.getResource(path, TableDesc.class, TABLE_SERIALIZER);
t.init();
String tableIdentity = t.getIdentity();
- checkNoDupName(tableIdentity, srcTableMap.containsKey(tableIdentity), "SourceTable", path);
srcTableMap.putLocal(tableIdentity, t);
return t;
}
+ public void reloadSourceTableExt(String tableIdentity) throws IOException {
+ reloadSourceTableExdAt(TableDesc.concatExdResourcePath(tableIdentity));
+ }
+
+
+ public void reloadSourceTable(String tableIdentity) throws IOException {
+ reloadSourceTableAt(TableDesc.concatResourcePath(tableIdentity));
+ }
+
+ public void reloadTableCache(String tableIdentity) throws IOException {
+ reloadSourceTableExt(tableIdentity);
+ reloadSourceTable(tableIdentity);
+ }
+
public DataModelDesc getDataModelDesc(String name) {
return dataModelDescMap.get(name);
}
@@ -297,24 +296,18 @@ public class MetadataManager {
private DataModelDesc reloadDataModelDescAt(String path) {
ResourceStore store = getStore();
- DataModelDesc ndesc = null;
try {
- ndesc = store.getResource(path, DataModelDesc.class, MODELDESC_SERIALIZER);
+ DataModelDesc dataModelDesc = store.getResource(path, DataModelDesc.class, MODELDESC_SERIALIZER);
+ dataModelDesc.init(this.getAllTablesMap());
+ if (dataModelDesc.getError().isEmpty() == false) {
+ throw new IllegalStateException("DataModelDesc at " + path + " has issues: " + dataModelDesc.getError());
+ }
+ String name = dataModelDesc.getName();
+ dataModelDescMap.putLocal(name, dataModelDesc);
+ return dataModelDesc;
} catch (IOException e) {
throw new IllegalStateException("Error to load" + path, e);
}
-
- ndesc.init(this.getAllTablesMap());
-
- if (ndesc.getError().isEmpty() == false) {
- throw new IllegalStateException("DataModelDesc at " + path + " has issues: " + ndesc.getError());
- }
-
- String name = ndesc.getName();
- checkNoDupName(name, dataModelDescMap.containsKey(name), "DataModel", path);
-
- dataModelDescMap.putLocal(name, ndesc);
- return ndesc;
}
public DataModelDesc createDataModelDesc(DataModelDesc dataModelDesc) throws IOException {
@@ -352,13 +345,6 @@ public class MetadataManager {
return dataModelDesc;
}
- public void deleteDataModelDesc(DataModelDesc dataModelDesc) throws IOException {
- // remove dataModelDesc
- String path = dataModelDesc.getResourcePath();
- getStore().deleteResource(path);
- dataModelDescMap.remove(dataModelDesc.getName());
- }
-
public void saveTableExd(String tableId, Map<String, String> tableExdProperties) throws IOException {
if (tableId == null) {
throw new IllegalArgumentException("tableId couldn't be null");
@@ -381,13 +367,4 @@ public class MetadataManager {
srcTableExdMap.putLocal(tableId, tableExdProperties);
}
- private void checkNoDupName(String name, boolean containsKey, String entityType, String path) {
- if (StringUtils.isBlank(name)) {
- throw new IllegalStateException(entityType + " name at " + path + ", must not be blank");
- }
- if (containsKey) {
- throw new IllegalStateException("Dup " + entityType + " name '" + name + "' at " + path);
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fe439a3d/server/src/main/java/com/kylinolap/rest/controller/CacheController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/com/kylinolap/rest/controller/CacheController.java b/server/src/main/java/com/kylinolap/rest/controller/CacheController.java
index 5a285cf..4b62013 100644
--- a/server/src/main/java/com/kylinolap/rest/controller/CacheController.java
+++ b/server/src/main/java/com/kylinolap/rest/controller/CacheController.java
@@ -65,16 +65,30 @@ public class CacheController extends BasicController {
public void wipeCache(@PathVariable String type, @PathVariable String event, @PathVariable String name) throws IOException {
Broadcaster.TYPE wipeType = Broadcaster.TYPE.getType(type);
EVENT wipeEvent = Broadcaster.EVENT.getEvent(event);
+ final String log = "wipe cache type: " + wipeType + " event:" + wipeEvent + " name:" + name;
+ logger.debug(log);
switch (wipeType) {
- case METADATA:
- logger.debug("Reload all metadata");
- cubeMgmtService.reloadMetadataCache();
- projectService.cleanDataCache();
- cubeMgmtService.cleanDataCache();
+ case TABLE:
+ switch (wipeEvent) {
+ case CREATE:
+ case UPDATE:
+ cubeMgmtService.getMetadataManager().reloadTableCache(name);
+ break;
+ case DROP:
+ throw new UnsupportedOperationException(log);
+ }
+ break;
+ case DATA_MODEL:
+ switch (wipeEvent) {
+ case CREATE:
+ case UPDATE:
+ cubeMgmtService.getMetadataManager().reloadDataModelDesc(name);
+ break;
+ case DROP:
+ throw new UnsupportedOperationException(log);
+ }
break;
case CUBE:
- logger.debug("Reload cube " + name + " with type:" + type + ", event type " + event);
- cubeMgmtService.reloadMetadataCache();
if ("ALL".equalsIgnoreCase(name.toUpperCase())) {
cubeMgmtService.cleanDataCache();
break;
@@ -91,8 +105,6 @@ public class CacheController extends BasicController {
}
break;
case PROJECT:
- logger.debug("Reload project " + name + " with type:" + type + ", event type " + event);
- cubeMgmtService.reloadMetadataCache();
if ("ALL".equalsIgnoreCase(name.toUpperCase())) {
projectService.cleanDataCache();
break;
@@ -109,7 +121,7 @@ public class CacheController extends BasicController {
}
break;
default:
- throw new RuntimeException("invalid type:" + wipeType);
+ throw new UnsupportedOperationException(log);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fe439a3d/server/src/main/java/com/kylinolap/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/com/kylinolap/rest/service/BasicService.java b/server/src/main/java/com/kylinolap/rest/service/BasicService.java
index 01ac023..2db6e82 100644
--- a/server/src/main/java/com/kylinolap/rest/service/BasicService.java
+++ b/server/src/main/java/com/kylinolap/rest/service/BasicService.java
@@ -131,14 +131,6 @@ public abstract class BasicService {
BasicService.resetOLAPDataSources();
}
- /**
- * Reload the cube desc with name {name} into cache
- *
- */
- public void reloadMetadataCache() {
- MetadataManager.getInstance(getConfig()).reload();
- }
-
public final KylinConfig getKylinConfig() {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fe439a3d/server/src/main/java/com/kylinolap/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/com/kylinolap/rest/service/CubeService.java b/server/src/main/java/com/kylinolap/rest/service/CubeService.java
index 77dba8c..6aa59b3 100644
--- a/server/src/main/java/com/kylinolap/rest/service/CubeService.java
+++ b/server/src/main/java/com/kylinolap/rest/service/CubeService.java
@@ -527,10 +527,9 @@ public class CubeService extends BasicService {
*
* @throws IOException
* @throws JobException
- * @throws UnknownHostException
* @throws CubeIntegrityException
*/
- private void releaseAllSegments(CubeInstance cube) throws IOException, JobException, UnknownHostException, CubeIntegrityException {
+ private void releaseAllSegments(CubeInstance cube) throws IOException, JobException, CubeIntegrityException {
final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null);
for (CubingJob cubingJob : cubingJobs) {
final ExecutableState status = cubingJob.getStatus();
@@ -545,7 +544,6 @@ public class CubeService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER)
public String[] reloadHiveTable(String tables) throws IOException {
Set<String> loaded = HiveSourceTableLoader.reloadHiveTables(tables.split(","), getConfig());
- getMetadataManager().reload();
return (String[]) loaded.toArray(new String[loaded.size()]);
}