You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/15 13:26:26 UTC
[10/50] [abbrv] incubator-kylin git commit: fix cache update policy
fix cache update policy
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1a7011ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1a7011ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1a7011ca
Branch: refs/heads/inverted-index
Commit: 1a7011ca9d14094e4fbed6f6c1da9723ee124b38
Parents: 84c9c35
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Jan 13 14:59:15 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Jan 13 14:59:15 2015 +0800
----------------------------------------------------------------------
.../common/restclient/AbstractRestCache.java | 2 +-
.../common/restclient/Broadcaster.java | 164 ++++++++-----------
.../rest/controller/CacheController.java | 4 +-
.../rest/interceptor/CacheIntercaptor.java | 42 -----
.../rest/interceptor/CacheInterceptor.java | 46 ++++++
5 files changed, 117 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1a7011ca/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java b/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
index 8bc2468..b5cc78e 100644
--- a/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
+++ b/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
@@ -30,7 +30,7 @@ public abstract class AbstractRestCache<K, V> {
protected final void syncRemote(Object key, Broadcaster.EVENT syncAction) {
String syncKey = (syncType == Broadcaster.TYPE.METADATA) ? "metadata" : key.toString();
- Broadcaster.queue(syncType.getType(), syncAction.getType(), syncKey);
+ Broadcaster.getInstance().queue(syncType.getType(), syncAction.getType(), syncKey);
}
public abstract void put(K key, V value);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1a7011ca/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java b/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
index 45013ea..695ecf3 100644
--- a/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
+++ b/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
@@ -22,7 +22,14 @@ import java.util.Date;
import java.util.List;
import java.util.Timer;
import java.util.TimerTask;
-
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,28 +45,55 @@ public class Broadcaster {
private static final Logger logger = LoggerFactory.getLogger(Broadcaster.class);
- private static List<BroadcastEvent> broadcaseEvents = new ArrayList<BroadcastEvent>();
+ private static BlockingDeque<BroadcastEvent> broadcaseEvents = new LinkedBlockingDeque<>();
static class BroadcasterHolder {
static final Broadcaster INSTANCE = new Broadcaster();
}
private Broadcaster() {
- Timer timer = new Timer();
- TimerTask task = new TimerTask() {
+ Executors.newSingleThreadExecutor().execute(new Runnable() {
+ @Override
public void run() {
- Broadcaster.flush();
+ final String[] nodes = KylinConfig.getInstanceFromEnv().getRestServers();
+ if (nodes == null || nodes.length < 1) {//TODO if the node count is greater than 1, it means it is a cluster
+ logger.warn("there are no available rest servers, please check the KYLIN_REST_SERVERS config");
+ return;
+ }
+ final List<RestClient> restClients = Lists.newArrayList();
+ for (String node: nodes) {
+ restClients.add(new RestClient(node));
+ }
+ final ExecutorService wipingCachePool = Executors.newFixedThreadPool(restClients.size());
+ while(true) {
+ try {
+ final BroadcastEvent broadcastEvent = broadcaseEvents.takeFirst();
+ logger.info("new broadcast event:" + broadcastEvent);
+ for (final RestClient restClient: restClients) {
+ wipingCachePool.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ restClient.wipeCache(broadcastEvent.getType(), broadcastEvent.getAction(), broadcastEvent.getName());
+ } catch (IOException e) {
+ logger.warn("Thread failed during wipe cache at " + broadcastEvent);
+ }
+ }
+ });
+ }
+ } catch (Exception e) {
+ logger.error("error running wiping", e);
+ }
+ }
}
- };
-
- timer.schedule(task, new Date(), 10 * 1000);
+ });
}
- public Broadcaster getInstance() {
+ public static Broadcaster getInstance() {
return BroadcasterHolder.INSTANCE;
}
- public static void queueSyncMetadata() {
+ public void queueSyncMetadata() {
queue(TYPE.METADATA.getType(), EVENT.CREATE.getType(), "metadata");
}
@@ -69,65 +103,20 @@ public class Broadcaster {
* @param action
* event action
*/
- public static synchronized void queue(String type, String action, String key) {
- BroadcastEvent event = BroadcasterHolder.INSTANCE.new BroadcastEvent(type, action, key);
-
- if (!broadcaseEvents.contains(event)) {
- broadcaseEvents.add(event);
+ public void queue(String type, String action, String key) {
+ try {
+ broadcaseEvents.putFirst(new BroadcastEvent(type, action, key));
+ } catch (Exception e) {
+ logger.error("error putting BroadcastEvent", e);
}
}
- public static synchronized void flush() {
- String[] nodes = KylinConfig.getInstanceFromEnv().getRestServers();
- if (nodes == null)
- return;
-
- for (BroadcastEvent event : broadcaseEvents) {
- for (String nodeUri : nodes) {
- logger.debug("Broadcast nodeUri: " + nodeUri + ", type: " + event.getType() + ", action: " + event.getAction() + ", name: " + event.getName());
- WipeCacheThread thread = BroadcasterHolder.INSTANCE.new WipeCacheThread(nodeUri, event.getType(), event.getAction(), event.getName());
- thread.start();
- }
- }
-
- broadcaseEvents.clear();
- }
-
public static String genEventkey(String type, String action, String name) {
String time = String.valueOf(System.currentTimeMillis());
return time + "_" + type + "_" + action + "_" + name;
}
- protected class WipeCacheThread extends Thread {
- private String nodeUri;
- private String type;
- private String action;
- private String name;
-
- public WipeCacheThread(String nodeUri, String type, String action, String name) {
- this.nodeUri = nodeUri;
- this.type = type;
- this.action = action;
- this.name = name;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Thread#run()
- */
- @Override
- public void run() {
- RestClient restClient = new RestClient(nodeUri);
- try {
- restClient.wipeCache(this.type, this.action, this.name);
- } catch (IOException e) {
- logger.warn("Thread failed during wipe cache at " + type + "." + action + "." + name + ", " + e.toString());
- }
- }
- }
-
- public enum EVENT {
+ public static enum EVENT {
CREATE("create"), UPDATE("update"), DROP("drop");
private String text;
@@ -150,7 +139,7 @@ public class Broadcaster {
}
}
- public enum TYPE {
+ public static enum TYPE {
CUBE("realization"), METADATA("metadata"), PROJECT("project");
private String text;
@@ -177,7 +166,7 @@ public class Broadcaster {
}
}
- public class BroadcastEvent {
+ public static class BroadcastEvent {
private String type;
private String action;
private String name;
@@ -193,31 +182,18 @@ public class Broadcaster {
return type;
}
- public void setType(String type) {
- this.type = type;
- }
-
public String getAction() {
return action;
}
- public void setAction(String action) {
- this.action = action;
- }
-
public String getName() {
return name;
}
- public void setName(String name) {
- this.name = name;
- }
-
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
- result = prime * result + getOuterType().hashCode();
result = prime * result + ((action == null) ? 0 : action.hashCode());
result = prime * result + ((name == null) ? 0 : name.hashCode());
result = prime * result + ((type == null) ? 0 : type.hashCode());
@@ -226,35 +202,31 @@ public class Broadcaster {
@Override
public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
+ if (obj == null) {
return false;
- if (getClass() != obj.getClass())
+ }
+ if (this == obj) {
+ return true;
+ }
+ if (getClass() != obj.getClass()) {
return false;
+ }
BroadcastEvent other = (BroadcastEvent) obj;
- if (!getOuterType().equals(other.getOuterType()))
+ if (!StringUtils.equals(action, other.action)) {
return false;
- if (action == null) {
- if (other.action != null)
- return false;
- } else if (!action.equals(other.action))
- return false;
- if (name == null) {
- if (other.name != null)
- return false;
- } else if (!name.equals(other.name))
+ }
+ if (!StringUtils.equals(name, other.name)) {
return false;
- if (type == null) {
- if (other.type != null)
- return false;
- } else if (!type.equals(other.type))
+ }
+ if (!StringUtils.equals(type, other.type)) {
return false;
+ }
return true;
}
- private Broadcaster getOuterType() {
- return Broadcaster.this;
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this).add("type", type).add("name", name).add("action", action).toString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1a7011ca/server/src/main/java/com/kylinolap/rest/controller/CacheController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/com/kylinolap/rest/controller/CacheController.java b/server/src/main/java/com/kylinolap/rest/controller/CacheController.java
index 9a870ca..0e37be5 100644
--- a/server/src/main/java/com/kylinolap/rest/controller/CacheController.java
+++ b/server/src/main/java/com/kylinolap/rest/controller/CacheController.java
@@ -53,9 +53,9 @@ public class CacheController extends BasicController {
* Wipe system cache
*
* @param type
- * {@link MetadataConstances.TYPE}
+ * {@link Broadcaster.TYPE}
* @param event
- * {@link MetadataConstances.EVENT}
+ * {@link Broadcaster.EVENT}
* @param name
* @return if the action success
* @throws IOException
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1a7011ca/server/src/main/java/com/kylinolap/rest/interceptor/CacheIntercaptor.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/com/kylinolap/rest/interceptor/CacheIntercaptor.java b/server/src/main/java/com/kylinolap/rest/interceptor/CacheIntercaptor.java
deleted file mode 100644
index 2b763dd..0000000
--- a/server/src/main/java/com/kylinolap/rest/interceptor/CacheIntercaptor.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.rest.interceptor;
-
-import org.aspectj.lang.JoinPoint;
-import org.aspectj.lang.annotation.After;
-import org.aspectj.lang.annotation.Aspect;
-import org.springframework.stereotype.Component;
-
-import com.kylinolap.common.restclient.Broadcaster;
-
-/**
- * @author xduo
- *
- */
-@Aspect
-@Component("cacheIntercaptor")
-public class CacheIntercaptor {
-
- @After("execution(public * com.kylinolap.rest.controller.CubeController.*(..)) || execution(public * com.kylinolap.rest.controller.ProjectController.*(..))")
- public void flush(JoinPoint joinPoint) {
- String methodName = joinPoint.getSignature().getName();
-
- if (methodName.matches("(update|create|save|disable|enable|delete|drop|purge)")) {
- Broadcaster.flush();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1a7011ca/server/src/main/java/com/kylinolap/rest/interceptor/CacheInterceptor.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/com/kylinolap/rest/interceptor/CacheInterceptor.java b/server/src/main/java/com/kylinolap/rest/interceptor/CacheInterceptor.java
new file mode 100644
index 0000000..0973a81
--- /dev/null
+++ b/server/src/main/java/com/kylinolap/rest/interceptor/CacheInterceptor.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.rest.interceptor;
+
+import org.aspectj.lang.JoinPoint;
+import org.aspectj.lang.annotation.After;
+import org.aspectj.lang.annotation.Aspect;
+import org.springframework.stereotype.Component;
+
+import com.kylinolap.common.restclient.Broadcaster;
+
+import java.util.regex.Pattern;
+
+/**
+ * @author xduo
+ *
+ */
+@Aspect
+@Component("cacheIntercaptor")
+public class CacheInterceptor {
+
+ private static Pattern UPDATE_PATTERN = Pattern.compile("(update|create|save|disable|enable|delete|drop|purge)");
+
+ @After("execution(public * com.kylinolap.rest.controller.CubeController.*(..)) || execution(public * com.kylinolap.rest.controller.ProjectController.*(..))")
+ public void flush(JoinPoint joinPoint) {
+ String methodName = joinPoint.getSignature().getName();
+
+ if (UPDATE_PATTERN.matcher(methodName).find()) {
+// Broadcaster.flush();
+ }
+ }
+}