You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/15 13:26:26 UTC

[10/50] [abbrv] incubator-kylin git commit: fix cache update policy

fix cache update policy


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1a7011ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1a7011ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1a7011ca

Branch: refs/heads/inverted-index
Commit: 1a7011ca9d14094e4fbed6f6c1da9723ee124b38
Parents: 84c9c35
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Jan 13 14:59:15 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Jan 13 14:59:15 2015 +0800

----------------------------------------------------------------------
 .../common/restclient/AbstractRestCache.java    |   2 +-
 .../common/restclient/Broadcaster.java          | 164 ++++++++-----------
 .../rest/controller/CacheController.java        |   4 +-
 .../rest/interceptor/CacheIntercaptor.java      |  42 -----
 .../rest/interceptor/CacheInterceptor.java      |  46 ++++++
 5 files changed, 117 insertions(+), 141 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1a7011ca/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java b/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
index 8bc2468..b5cc78e 100644
--- a/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
+++ b/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
@@ -30,7 +30,7 @@ public abstract class AbstractRestCache<K, V> {
 
     protected final void syncRemote(Object key, Broadcaster.EVENT syncAction) {
         String syncKey = (syncType == Broadcaster.TYPE.METADATA) ? "metadata" : key.toString();
-        Broadcaster.queue(syncType.getType(), syncAction.getType(), syncKey);
+        Broadcaster.getInstance().queue(syncType.getType(), syncAction.getType(), syncKey);
     }
 
     public abstract void put(K key, V value);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1a7011ca/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java b/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
index 45013ea..695ecf3 100644
--- a/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
+++ b/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
@@ -22,7 +22,14 @@ import java.util.Date;
 import java.util.List;
 import java.util.Timer;
 import java.util.TimerTask;
-
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -38,28 +45,55 @@ public class Broadcaster {
 
     private static final Logger logger = LoggerFactory.getLogger(Broadcaster.class);
 
-    private static List<BroadcastEvent> broadcaseEvents = new ArrayList<BroadcastEvent>();
+    private static BlockingDeque<BroadcastEvent> broadcaseEvents = new LinkedBlockingDeque<>();
 
     static class BroadcasterHolder {
         static final Broadcaster INSTANCE = new Broadcaster();
     }
 
     private Broadcaster() {
-        Timer timer = new Timer();
-        TimerTask task = new TimerTask() {
+        Executors.newSingleThreadExecutor().execute(new Runnable() {
+            @Override
             public void run() {
-                Broadcaster.flush();
+                final String[] nodes = KylinConfig.getInstanceFromEnv().getRestServers();
+                if (nodes == null || nodes.length < 1) {//TODO if the node count is greater than 1, it means it is a cluster
+                    logger.warn("there are no available rest servers, please check the KYLIN_REST_SERVERS config");
+                    return;
+                }
+                final List<RestClient> restClients = Lists.newArrayList();
+                for (String node: nodes) {
+                    restClients.add(new RestClient(node));
+                }
+                final ExecutorService wipingCachePool = Executors.newFixedThreadPool(restClients.size());
+                while(true) {
+                    try {
+                        final BroadcastEvent broadcastEvent = broadcaseEvents.takeFirst();
+                        logger.info("new broadcast event:" + broadcastEvent);
+                        for (final RestClient restClient: restClients) {
+                            wipingCachePool.execute(new Runnable() {
+                                @Override
+                                public void run() {
+                                    try {
+                                        restClient.wipeCache(broadcastEvent.getType(), broadcastEvent.getAction(), broadcastEvent.getName());
+                                    } catch (IOException e) {
+                                        logger.warn("Thread failed during wipe cache at " + broadcastEvent);
+                                    }
+                                }
+                            });
+                        }
+                    } catch (Exception e) {
+                        logger.error("error running wiping", e);
+                    }
+                }
             }
-        };
-
-        timer.schedule(task, new Date(), 10 * 1000);
+        });
     }
 
-    public Broadcaster getInstance() {
+    public static Broadcaster getInstance() {
         return BroadcasterHolder.INSTANCE;
     }
 
-    public static void queueSyncMetadata() {
+    public void queueSyncMetadata() {
         queue(TYPE.METADATA.getType(), EVENT.CREATE.getType(), "metadata");
     }
 
@@ -69,65 +103,20 @@ public class Broadcaster {
      * @param action
      *            event action
      */
-    public static synchronized void queue(String type, String action, String key) {
-        BroadcastEvent event = BroadcasterHolder.INSTANCE.new BroadcastEvent(type, action, key);
-
-        if (!broadcaseEvents.contains(event)) {
-            broadcaseEvents.add(event);
+    public void queue(String type, String action, String key) {
+        try {
+            broadcaseEvents.putFirst(new BroadcastEvent(type, action, key));
+        } catch (Exception e) {
+            logger.error("error putting BroadcastEvent", e);
         }
     }
 
-    public static synchronized void flush() {
-        String[] nodes = KylinConfig.getInstanceFromEnv().getRestServers();
-        if (nodes == null)
-            return;
-
-        for (BroadcastEvent event : broadcaseEvents) {
-            for (String nodeUri : nodes) {
-                logger.debug("Broadcast nodeUri: " + nodeUri + ", type: " + event.getType() + ", action: " + event.getAction() + ", name: " + event.getName());
-                WipeCacheThread thread = BroadcasterHolder.INSTANCE.new WipeCacheThread(nodeUri, event.getType(), event.getAction(), event.getName());
-                thread.start();
-            }
-        }
-
-        broadcaseEvents.clear();
-    }
-
     public static String genEventkey(String type, String action, String name) {
         String time = String.valueOf(System.currentTimeMillis());
         return time + "_" + type + "_" + action + "_" + name;
     }
 
-    protected class WipeCacheThread extends Thread {
-        private String nodeUri;
-        private String type;
-        private String action;
-        private String name;
-
-        public WipeCacheThread(String nodeUri, String type, String action, String name) {
-            this.nodeUri = nodeUri;
-            this.type = type;
-            this.action = action;
-            this.name = name;
-        }
-
-        /*
-         * (non-Javadoc)
-         * 
-         * @see java.lang.Thread#run()
-         */
-        @Override
-        public void run() {
-            RestClient restClient = new RestClient(nodeUri);
-            try {
-                restClient.wipeCache(this.type, this.action, this.name);
-            } catch (IOException e) {
-                logger.warn("Thread failed during wipe cache at " + type + "." + action + "." + name + ", " + e.toString());
-            }
-        }
-    }
-
-    public enum EVENT {
+    public static enum EVENT {
         CREATE("create"), UPDATE("update"), DROP("drop");
         private String text;
 
@@ -150,7 +139,7 @@ public class Broadcaster {
         }
     }
 
-    public enum TYPE {
+    public static enum TYPE {
         CUBE("realization"), METADATA("metadata"), PROJECT("project");
         private String text;
 
@@ -177,7 +166,7 @@ public class Broadcaster {
         }
     }
 
-    public class BroadcastEvent {
+    public static class BroadcastEvent {
         private String type;
         private String action;
         private String name;
@@ -193,31 +182,18 @@ public class Broadcaster {
             return type;
         }
 
-        public void setType(String type) {
-            this.type = type;
-        }
-
         public String getAction() {
             return action;
         }
 
-        public void setAction(String action) {
-            this.action = action;
-        }
-
         public String getName() {
             return name;
         }
 
-        public void setName(String name) {
-            this.name = name;
-        }
-
         @Override
         public int hashCode() {
             final int prime = 31;
             int result = 1;
-            result = prime * result + getOuterType().hashCode();
             result = prime * result + ((action == null) ? 0 : action.hashCode());
             result = prime * result + ((name == null) ? 0 : name.hashCode());
             result = prime * result + ((type == null) ? 0 : type.hashCode());
@@ -226,35 +202,31 @@ public class Broadcaster {
 
         @Override
         public boolean equals(Object obj) {
-            if (this == obj)
-                return true;
-            if (obj == null)
+            if (obj == null) {
                 return false;
-            if (getClass() != obj.getClass())
+            }
+            if (this == obj) {
+                return true;
+            }
+            if (getClass() != obj.getClass()) {
                 return false;
+            }
             BroadcastEvent other = (BroadcastEvent) obj;
-            if (!getOuterType().equals(other.getOuterType()))
+            if (!StringUtils.equals(action, other.action)) {
                 return false;
-            if (action == null) {
-                if (other.action != null)
-                    return false;
-            } else if (!action.equals(other.action))
-                return false;
-            if (name == null) {
-                if (other.name != null)
-                    return false;
-            } else if (!name.equals(other.name))
+            }
+            if (!StringUtils.equals(name, other.name)) {
                 return false;
-            if (type == null) {
-                if (other.type != null)
-                    return false;
-            } else if (!type.equals(other.type))
+            }
+            if (!StringUtils.equals(type, other.type)) {
                 return false;
+            }
             return true;
         }
 
-        private Broadcaster getOuterType() {
-            return Broadcaster.this;
+        @Override
+        public String toString() {
+            return Objects.toStringHelper(this).add("type", type).add("name", name).add("action", action).toString();
         }
 
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1a7011ca/server/src/main/java/com/kylinolap/rest/controller/CacheController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/com/kylinolap/rest/controller/CacheController.java b/server/src/main/java/com/kylinolap/rest/controller/CacheController.java
index 9a870ca..0e37be5 100644
--- a/server/src/main/java/com/kylinolap/rest/controller/CacheController.java
+++ b/server/src/main/java/com/kylinolap/rest/controller/CacheController.java
@@ -53,9 +53,9 @@ public class CacheController extends BasicController {
      * Wipe system cache
      * 
      * @param type
-     *            {@link MetadataConstances.TYPE}
+     *            {@link Broadcaster.TYPE}
      * @param event
-     *            {@link MetadataConstances.EVENT}
+     *            {@link Broadcaster.EVENT}
      * @param name
      * @return if the action success
      * @throws IOException

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1a7011ca/server/src/main/java/com/kylinolap/rest/interceptor/CacheIntercaptor.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/com/kylinolap/rest/interceptor/CacheIntercaptor.java b/server/src/main/java/com/kylinolap/rest/interceptor/CacheIntercaptor.java
deleted file mode 100644
index 2b763dd..0000000
--- a/server/src/main/java/com/kylinolap/rest/interceptor/CacheIntercaptor.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.rest.interceptor;
-
-import org.aspectj.lang.JoinPoint;
-import org.aspectj.lang.annotation.After;
-import org.aspectj.lang.annotation.Aspect;
-import org.springframework.stereotype.Component;
-
-import com.kylinolap.common.restclient.Broadcaster;
-
-/**
- * @author xduo
- * 
- */
-@Aspect
-@Component("cacheIntercaptor")
-public class CacheIntercaptor {
-
-    @After("execution(public * com.kylinolap.rest.controller.CubeController.*(..)) || execution(public * com.kylinolap.rest.controller.ProjectController.*(..))")
-    public void flush(JoinPoint joinPoint) {
-        String methodName = joinPoint.getSignature().getName();
-
-        if (methodName.matches("(update|create|save|disable|enable|delete|drop|purge)")) {
-            Broadcaster.flush();
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1a7011ca/server/src/main/java/com/kylinolap/rest/interceptor/CacheInterceptor.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/com/kylinolap/rest/interceptor/CacheInterceptor.java b/server/src/main/java/com/kylinolap/rest/interceptor/CacheInterceptor.java
new file mode 100644
index 0000000..0973a81
--- /dev/null
+++ b/server/src/main/java/com/kylinolap/rest/interceptor/CacheInterceptor.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.rest.interceptor;
+
+import org.aspectj.lang.JoinPoint;
+import org.aspectj.lang.annotation.After;
+import org.aspectj.lang.annotation.Aspect;
+import org.springframework.stereotype.Component;
+
+import com.kylinolap.common.restclient.Broadcaster;
+
+import java.util.regex.Pattern;
+
+/**
+ * @author xduo
+ * 
+ */
+@Aspect
+@Component("cacheIntercaptor")
+public class CacheInterceptor {
+
+    private static Pattern UPDATE_PATTERN = Pattern.compile("(update|create|save|disable|enable|delete|drop|purge)");
+
+    @After("execution(public * com.kylinolap.rest.controller.CubeController.*(..)) || execution(public * com.kylinolap.rest.controller.ProjectController.*(..))")
+    public void flush(JoinPoint joinPoint) {
+        String methodName = joinPoint.getSignature().getName();
+
+        if (UPDATE_PATTERN.matcher(methodName).find()) {
+//            Broadcaster.flush();
+        }
+    }
+}