You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/09/24 06:30:50 UTC
kylin git commit: KYLIN-2033 code review and bug fixes
Repository: kylin
Updated Branches:
refs/heads/master afa350f00 -> bf127a916
KYLIN-2033 code review and bug fixes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bf127a91
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bf127a91
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bf127a91
Branch: refs/heads/master
Commit: bf127a9167c506025616031374a91c57eb1355ed
Parents: afa350f
Author: Yang Li <li...@apache.org>
Authored: Sat Sep 24 14:29:39 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Sep 24 14:29:56 2016 +0800
----------------------------------------------------------------------
.../kylin/metadata/cachesync/Broadcaster.java | 4 ++--
.../realization/RealizationRegistry.java | 8 ++++++++
.../kylin/rest/controller/CacheController.java | 17 +++++++++-------
.../apache/kylin/rest/service/AdminService.java | 1 -
.../apache/kylin/rest/service/CacheService.java | 21 ++++++++++++++++----
.../apache/kylin/rest/service/QueryService.java | 4 +---
webapp/app/js/services/cache.js | 2 +-
7 files changed, 39 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf127a91/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index 380dac8..75b2333 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -112,7 +112,7 @@ public class Broadcaster {
while (true) {
try {
final BroadcastEvent broadcastEvent = broadcastEvents.takeFirst();
- logger.info("Announcing new broadcast event:" + broadcastEvent);
+ logger.info("Announcing new broadcast event: " + broadcastEvent);
for (final RestClient restClient : restClients) {
wipingCachePool.execute(new Runnable() {
@Override
@@ -326,7 +326,7 @@ public class Broadcaster {
@Override
public String toString() {
- return Objects.toStringHelper(this).add("type", entity).add("name", cacheKey).add("action", event).toString();
+ return Objects.toStringHelper(this).add("entity", entity).add("event", event).add("cacheKey", cacheKey).toString();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf127a91/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
index e6b1a86..77e2679 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
@@ -26,6 +26,7 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,6 +76,13 @@ public class RealizationRegistry {
logger.info("Initializing RealizationRegistry with metadata url " + config);
this.config = config;
init();
+
+ Broadcaster.getInstance(config).registerListener(new Broadcaster.Listener() {
+ @Override
+ public void onClearAll(Broadcaster broadcaster) throws IOException {
+ clearCache();
+ }
+ }, "");
}
private void init() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf127a91/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
index 667046b..254aabf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
@@ -47,13 +47,16 @@ public class CacheController extends BasicController {
private CacheService cacheService;
/**
- * Wipe system cache
- *
- * @param entity {@link Broadcaster.TYPE}
- * @param event {@link Broadcaster.Event}
- * @param cacheKey
- * @return if the action success
- * @throws IOException
+ * Announce wipe cache to all cluster nodes
+ */
+ @RequestMapping(value = "/announce/{entity}/{cacheKey}/{event}", method = { RequestMethod.PUT })
+ @ResponseBody
+ public void announceWipeCache(@PathVariable String entity, @PathVariable String event, @PathVariable String cacheKey) throws IOException {
+ cacheService.annouceWipeCache(entity, event, cacheKey);
+ }
+
+ /**
+ * Wipe cache on this node
*/
@RequestMapping(value = "/{entity}/{cacheKey}/{event}", method = { RequestMethod.PUT })
@ResponseBody
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf127a91/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
index b92c305..1a94967 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
@@ -48,7 +48,6 @@ public class AdminService extends BasicService {
*/
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public String getEnv() {
- logger.debug("Get Kylin Runtime environment");
PropertiesConfiguration tempConfig = new PropertiesConfiguration();
// Add Java Env
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf127a91/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 5d29dcd..c9c2dd7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -59,15 +59,23 @@ public class CacheService extends BasicService {
@Autowired
private CacheManager cacheManager;
-
+
private Broadcaster.Listener cacheSyncListener = new Broadcaster.Listener() {
@Override
+ public void onClearAll(Broadcaster broadcaster) throws IOException {
+ removeAllOLAPDataSources();
+ cleanAllDataCache();
+ }
+
+ @Override
public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException {
removeOLAPDataSource(project);
+ cleanDataCache(project);
}
@Override
public void onProjectDataChange(Broadcaster broadcaster, String project) throws IOException {
+ removeOLAPDataSource(project); // data availability (cube enabled/disabled) affects exposed schema to SQL
cleanDataCache(project);
}
@@ -84,7 +92,7 @@ public class CacheService extends BasicService {
logger.error("Error in updateOnNewSegmentReady()", ex);
}
}
- }.run();
+ }.start();
}
}
};
@@ -94,12 +102,17 @@ public class CacheService extends BasicService {
this.cubeService = cubeService;
}
+ public void annouceWipeCache(String entity, String event, String cacheKey) {
+ Broadcaster broadcaster = Broadcaster.getInstance(getConfig());
+ broadcaster.queue(entity, event, cacheKey);
+ }
+
public void notifyMetadataChange(String entity, Event event, String cacheKey) throws IOException {
Broadcaster broadcaster = Broadcaster.getInstance(getConfig());
-
+
// broadcaster can be clearCache() too, make sure listener is registered; re-registration will be ignored
broadcaster.registerListener(cacheSyncListener, "cube");
-
+
broadcaster.notifyListener(entity, event, cacheKey);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf127a91/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index df296cf..8303cee 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -337,7 +337,6 @@ public class QueryService extends BasicService {
conn = dataSource.getConnection();
DatabaseMetaData metaData = conn.getMetaData();
- logger.debug("getting table metas");
JDBCTableMeta = metaData.getTables(null, null, null, null);
tableMetas = new LinkedList<TableMeta>();
@@ -355,7 +354,6 @@ public class QueryService extends BasicService {
}
}
- logger.debug("getting column metas");
columnMeta = metaData.getColumns(null, null, null, null);
while (columnMeta.next()) {
@@ -369,7 +367,7 @@ public class QueryService extends BasicService {
tableMap.get(colmnMeta.getTABLE_SCHEM() + "#" + colmnMeta.getTABLE_NAME()).addColumn(colmnMeta);
}
}
- logger.debug("done column metas");
+
} finally {
close(columnMeta, null, conn);
if (JDBCTableMeta != null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf127a91/webapp/app/js/services/cache.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/cache.js b/webapp/app/js/services/cache.js
index b61a179..bcec603 100644
--- a/webapp/app/js/services/cache.js
+++ b/webapp/app/js/services/cache.js
@@ -17,7 +17,7 @@
*/
KylinApp.factory('CacheService', ['$resource', function ($resource, config) {
- return $resource(Config.service.url + 'cache/:type/:name/:action', {}, {
+ return $resource(Config.service.url + 'cache/announce/:type/:name/:action', {}, {
clean: {method: 'PUT', params: {type: 'all', name: 'all', action: 'update'}, isArray: false}
});
}]);