You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/08/20 09:36:18 UTC

[1/2] kylin git commit: KYLIN-2716: remove non-thread-safe WeakHashMap for ClassUtil

Repository: kylin
Updated Branches:
  refs/heads/master 5b942267f -> 807395787


KYLIN-2716: remove non-thread-safe WeakHashMap for ClassUtil

Signed-off-by: Li Yang <li...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bdb3a5fd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bdb3a5fd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bdb3a5fd

Branch: refs/heads/master
Commit: bdb3a5fde7635ab3601abdaf6c2e2653ccd5ac60
Parents: 5b94226
Author: Zhong <nj...@apache.org>
Authored: Wed Aug 2 15:37:44 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Aug 20 17:24:35 2017 +0800

----------------------------------------------------------------------
 .../java/org/apache/kylin/common/util/ClassUtil.java   | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/bdb3a5fd/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
index 0eb1af5..a78cb48 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
@@ -27,7 +27,6 @@ import java.net.URLDecoder;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.WeakHashMap;
 
 import org.slf4j.LoggerFactory;
 
@@ -54,7 +53,6 @@ public class ClassUtil {
         }
     }
 
-    private static final WeakHashMap<String, Class<?>> forNameCache = new WeakHashMap<>();
     private static final Map<String, String> classRenameMap;
     static {
         classRenameMap = new HashMap<>();
@@ -71,15 +69,8 @@ public class ClassUtil {
 
     @SuppressWarnings("unchecked")
     public static <T> Class<? extends T> forName(String name, Class<T> clz) throws ClassNotFoundException {
-        String origName = name;
-
-        Class<? extends T> result = (Class<? extends T>) forNameCache.get(origName);
-        if (result == null) {
-            name = forRenamedClass(name);
-            result = (Class<? extends T>) Class.forName(name);
-            forNameCache.put(origName, result);
-        }
-        return result;
+        name = forRenamedClass(name);
+        return (Class<? extends T>) Class.forName(name);
     }
 
     private static String forRenamedClass(String name) {


[2/2] kylin git commit: KYLIN-2716: replace non-thread-safe WeakHashMap with Guava Cache for htableInfoCache in CubeService

Posted by li...@apache.org.
KYLIN-2716: replace non-thread-safe WeakHashMap with Guava Cache for htableInfoCache in CubeService

Signed-off-by: Li Yang <li...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/80739578
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/80739578
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/80739578

Branch: refs/heads/master
Commit: 8073957873e114a2a1152f73e4942be66730761e
Parents: bdb3a5f
Author: Zhong <nj...@apache.org>
Authored: Wed Aug 2 16:27:39 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Aug 20 17:30:48 2017 +0800

----------------------------------------------------------------------
 .../kylin/rest/controller/CubeController.java   |  3 +-
 .../rest/controller2/CubeControllerV2.java      |  3 +-
 .../apache/kylin/rest/service/CubeService.java  | 99 +++++++++++++++-----
 3 files changed, 81 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/80739578/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index a370292..a2cf0fb 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -27,6 +27,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.concurrent.ExecutionException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.util.JsonUtil;
@@ -553,7 +554,7 @@ public class CubeController extends BasicController {
             // Get info of given table.
             try {
                 hr = cubeService.getHTableInfo(tableName);
-            } catch (IOException e) {
+            } catch (IOException | ExecutionException e) {
                 logger.error("Failed to calcuate size of HTable \"" + tableName + "\".", e);
             }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/80739578/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
index 9ffc062..aba2cf9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
 
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
@@ -445,7 +446,7 @@ public class CubeControllerV2 extends BasicController {
             // Get info of given table.
             try {
                 hr = cubeService.getHTableInfo(tableName);
-            } catch (IOException e) {
+            } catch (IOException | ExecutionException e) {
                 logger.error("Failed to calculate size of HTable \"" + tableName + "\".", e);
             }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/80739578/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index aa42cb0..c32133d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -24,7 +24,8 @@ import java.util.Collections;
 import java.util.Date;
 import java.util.EnumSet;
 import java.util.List;
-import java.util.WeakHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.KylinConfig;
@@ -41,6 +42,7 @@ import org.apache.kylin.engine.mr.CubingJob;
 import org.apache.kylin.job.exception.JobException;
 import org.apache.kylin.job.execution.DefaultChainedExecutable;
 import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.metadata.cachesync.Broadcaster;
 import org.apache.kylin.metadata.draft.Draft;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
@@ -52,6 +54,7 @@ import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.rest.constant.Constant;
 import org.apache.kylin.rest.exception.BadRequestException;
 import org.apache.kylin.rest.exception.ForbiddenException;
+import org.apache.kylin.rest.exception.InternalErrorException;
 import org.apache.kylin.rest.msg.Message;
 import org.apache.kylin.rest.msg.MsgPicker;
 import org.apache.kylin.rest.request.MetricsRequest;
@@ -61,6 +64,7 @@ import org.apache.kylin.rest.security.AclPermission;
 import org.apache.kylin.rest.util.AclUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.security.access.AccessDeniedException;
@@ -69,6 +73,11 @@ import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Component;
 
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
 import com.google.common.collect.Lists;
 
 /**
@@ -77,13 +86,13 @@ import com.google.common.collect.Lists;
  * @author yangli9
  */
 @Component("cubeMgmtService")
-public class CubeService extends BasicService {
+public class CubeService extends BasicService implements InitializingBean {
 
     private static final Logger logger = LoggerFactory.getLogger(CubeService.class);
 
     public static final char[] VALID_CUBENAME = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_".toCharArray();
 
-    private WeakHashMap<String, HBaseResponse> htableInfoCache = new WeakHashMap<>();
+    private static final int CACHE_MAX_SIZE = 5000;
 
     @Autowired
     @Qualifier("accessService")
@@ -100,6 +109,69 @@ public class CubeService extends BasicService {
     @Autowired
     private AclUtil aclUtil;
 
+    /*
+    * (non-Javadoc)
+    *
+    * @see
+    * org.springframework.beans.factory.InitializingBean#afterPropertiesSet()
+    */
+    @SuppressWarnings("unchecked")
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        Broadcaster.getInstance(getConfig()).registerListener(new HTableInfoSyncListener(), "cube");
+    }
+
+    private final LoadingCache<String, HBaseResponse> htableInfoCache = CacheBuilder.newBuilder()
+            .maximumSize(CACHE_MAX_SIZE).expireAfterAccess(7, TimeUnit.DAYS)
+            .removalListener(new RemovalListener<String, HBaseResponse>() {
+                @Override
+                public void onRemoval(RemovalNotification<String, HBaseResponse> notification) {
+                    logger.info("Hbase table: " + notification.getKey() + " cache is removed due to "
+                            + notification.getCause());
+                }
+            }).build(new CacheLoader<String, HBaseResponse>() {
+                @Override
+                public HBaseResponse load(String tableName) throws Exception {
+                    HBaseResponse hr = new HBaseResponse();
+                    if ("hbase".equals(getConfig().getMetadataUrl().getScheme())) {
+                        try {
+                            // use reflection to isolate NoClassDef errors when HBase is not available
+                            hr = (HBaseResponse) Class.forName("org.apache.kylin.rest.service.HBaseInfoUtil")//
+                                    .getMethod("getHBaseInfo", new Class[] { String.class, String.class })//
+                                    .invoke(null, new Object[] { tableName, getConfig().getStorageUrl() });
+                        } catch (Throwable e) {
+                            throw new IOException(e);
+                        }
+                    }
+                    return hr;
+                }
+            });
+
+    private class HTableInfoSyncListener extends Broadcaster.Listener {
+        @Override
+        public void onClearAll(Broadcaster broadcaster) throws IOException {
+            htableInfoCache.invalidateAll();
+        }
+
+        @Override
+        public void onEntityChange(Broadcaster broadcaster, String entity, Broadcaster.Event event, String cacheKey)
+                throws IOException {
+            String cubeName = cacheKey;
+
+            CubeInstance cube = getCubeManager().getCube(cubeName);
+            if (null == cube) {
+                throw new InternalErrorException("Cannot find cube " + cubeName);
+            }
+
+            List<String> htableNameList = Lists.newArrayListWithExpectedSize(cube.getSegments().size());
+            for (CubeSegment segment : cube.getSegments()) {
+                htableNameList.add(segment.getStorageLocationIdentifier());
+            }
+
+            htableInfoCache.invalidateAll(htableNameList);
+        }
+    }
+
     @PostFilter(Constant.ACCESS_POST_FILTER_READ)
     public List<CubeInstance> listAllCubes(final String cubeName, final String projectName, final String modelName, boolean exactMatch) {
         List<CubeInstance> cubeInstances = null;
@@ -418,25 +490,8 @@ public class CubeService extends BasicService {
      * if error happens
      * @throws IOException Exception when HTable resource is not closed correctly.
      */
-    public HBaseResponse getHTableInfo(String tableName) throws IOException {
-        if (htableInfoCache.containsKey(tableName)) {
-            return htableInfoCache.get(tableName);
-        }
-
-        HBaseResponse hr = new HBaseResponse();
-        if ("hbase".equals(getConfig().getMetadataUrl().getScheme())) {
-            try {
-                // use reflection to isolate NoClassDef errors when HBase is not available
-                hr = (HBaseResponse) Class.forName("org.apache.kylin.rest.service.HBaseInfoUtil")//
-                        .getMethod("getHBaseInfo", new Class[] { String.class, String.class })//
-                        .invoke(null, new Object[] { tableName, this.getConfig().getStorageUrl() });
-            } catch (Throwable e) {
-                throw new IOException(e);
-            }
-        }
-
-        htableInfoCache.put(tableName, hr);
-        return hr;
+    public HBaseResponse getHTableInfo(String tableName) throws IOException, ExecutionException {
+        return htableInfoCache.get(tableName);
     }
 
     @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN