You are viewing a plain text version of this content. The canonical link for it is here.
Posted to gitbox@hive.apache.org by GitBox <gi...@apache.org> on 2020/07/16 16:54:51 UTC
[GitHub] [hive] soumyakanti3578 opened a new pull request #1265: [DRAFT] Support for HS2 client metadata cache
soumyakanti3578 opened a new pull request #1265:
URL: https://github.com/apache/hive/pull/1265
## NOTICE
This is a draft pull request
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] soumyakanti3578 closed pull request #1265: [DRAFT] Support for HS2 client metadata cache
Posted by GitBox <gi...@apache.org>.
soumyakanti3578 closed pull request #1265:
URL: https://github.com/apache/hive/pull/1265
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] belugabehr commented on a change in pull request #1265: [DRAFT] Support for HS2 client metadata cache
Posted by GitBox <gi...@apache.org>.
belugabehr commented on a change in pull request #1265:
URL: https://github.com/apache/hive/pull/1265#discussion_r455955778
##########
File path: standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientWithLocalCache.java
##########
@@ -0,0 +1,179 @@
+package org.apache.hadoop.hive.metastore;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.PartitionsByExprRequest;
+import org.apache.hadoop.hive.metastore.api.PartitionsByExprResult;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.ql.util.IncrementalObjectSizeEstimator;
+import org.apache.hadoop.hive.ql.util.IncrementalObjectSizeEstimator.ObjectEstimator;
+import org.apache.thrift.TException;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Objects;
+
+public class HiveMetaStoreClientWithLocalCache extends HiveMetaStoreClient {
+
+ private static LoadingCache<CacheKey, Object> mscLocalCache;
+ private static volatile boolean cacheInitialized = false;
+ private boolean isCacheEnabled = true;
+ private static HashMap<Class<?>, ObjectEstimator> sizeEstimator = null;
+
+ public HiveMetaStoreClientWithLocalCache(Configuration conf) throws MetaException {
+ this(conf, null, true);
+ }
+
+ public HiveMetaStoreClientWithLocalCache(Configuration conf, HiveMetaHookLoader hookLoader) throws MetaException {
+ this(conf, hookLoader, true);
+ }
+
+ public HiveMetaStoreClientWithLocalCache(Configuration conf, HiveMetaHookLoader hookLoader, Boolean allowEmbedded) throws MetaException {
+ super(conf, hookLoader, allowEmbedded);
+
+// isCacheEnabled = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.MSC_CACHE_ENABLED);
+ if (isCacheEnabled) {
+ // if cache is not initialized, init size estimator and the cache.
+ if (!cacheInitialized) {
+ cacheInitialized = true;
+ LOG.debug("Initializing local cache in HiveMetaStoreClient...");
+ // initialize a size estimator with this class and all other classes in KeyType.
+ initSizeEstimator();
+ initCache();
+ LOG.debug("Local cache initialized in HiveMetaStoreClient: " + mscLocalCache);
+ }
+ }
+ }
+
+ private static void initSizeEstimator() {
+ sizeEstimator = IncrementalObjectSizeEstimator.createEstimators(HiveMetaStoreClientWithLocalCache.class);
+ Arrays.stream(KeyType.values()).forEach(e -> {
+ IncrementalObjectSizeEstimator.createEstimators(e.keyClass, sizeEstimator);
+ IncrementalObjectSizeEstimator.createEstimators(e.valueClass, sizeEstimator);}
+ );
+ }
+
+ public enum KeyType {
+ PARTITIONS_BY_EXPR_REQUEST(PartitionsByExprRequest.class, PartitionsByExprResult.class);
+
+ private final Class<?> keyClass;
+ private final Class<?> valueClass;
+
+ KeyType(Class<?> keyClass, Class<?> valueClass) {
+ this.keyClass = keyClass;
+ this.valueClass = valueClass;
+ }
+ }
+
+ public static class CacheKey{
+ KeyType IDENTIFIER;
+ Object obj;
+
+ public CacheKey(KeyType IDENTIFIER, Object obj) {
+ this.IDENTIFIER = IDENTIFIER;
+ this.obj = obj;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ CacheKey cacheKey = (CacheKey) o;
+ return IDENTIFIER == cacheKey.IDENTIFIER &&
+ Objects.equals(obj, cacheKey.obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(IDENTIFIER, obj);
+ }
+ }
+
+ private static int getWeight(CacheKey key, Object val) {
+ if (val instanceof Exception) return 0;
+ ObjectEstimator keySizeEstimator = sizeEstimator.get(key.IDENTIFIER.keyClass);
+ ObjectEstimator valSizeEstimator = sizeEstimator.get(key.IDENTIFIER.valueClass);
+ int keySize = keySizeEstimator.estimate(key, sizeEstimator);
+ int valSize = valSizeEstimator.estimate(val, sizeEstimator);
+ LOG.debug("Cache entry weight - key: {}, value: {}, total: {}", keySize, valSize, keySize+valSize);
+ return keySize + valSize;
+ }
+
+ private Object getOrLoad(CacheKey key) {
+ Object val;
+ try {
+ val = getResultObject(key);
+ } catch (Exception e) {
+ LOG.debug("Exception in MSC local cache: " + e.toString());
+ if (e instanceof MetaException) {
+ val = new MetaException(e.getMessage());
+ } else {
+ val = new Exception(e.getMessage());
+ }
+ }
+ return val;
+ }
+
+ private synchronized void initCache() {
+ int maxSize = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.MSC_CACHE_MAX_SIZE);
+ int initSize = 100;
+ mscLocalCache = Caffeine.newBuilder()
+ .initialCapacity(initSize)
+ .maximumWeight(maxSize)
+ .weigher(HiveMetaStoreClientWithLocalCache::getWeight)
+ .removalListener((key, val, cause) ->
+ LOG.debug(String.format("Caffeine - (%s, %s) was removed (%s)", key, val, cause)))
+ .recordStats()
+ .build(this::getOrLoad);
+ }
+
+
+ protected Object getResultObject(CacheKey cacheKey) throws TException {
+ Object result = null;
+
+ switch (cacheKey.IDENTIFIER) {
+ case PARTITIONS_BY_EXPR_REQUEST:
+ result = super.getPartitionsByExprResult((PartitionsByExprRequest)cacheKey.obj);
+ break;
+ default:
+ break;
+ }
+
+ return result;
+ }
+
+ @Override
+ protected PartitionsByExprResult getPartitionsByExprResult(PartitionsByExprRequest req) throws TException {
+ PartitionsByExprResult r = null;
+
+ if (isCacheEnabled) {
+ CacheKey cacheKey = new CacheKey(KeyType.PARTITIONS_BY_EXPR_REQUEST, req);
+ Object val;
+ try {
+ val = mscLocalCache.get(cacheKey); // get either the result or an Exception
+
+ if (val instanceof PartitionsByExprResult) {
+ r = (PartitionsByExprResult) val;
+ LOG.debug(mscLocalCache.toString().substring(mscLocalCache.toString().indexOf("LoadingCache")) + ": " +
Review comment:
Not sure why this needs to substring, but if this can't be kept as simple as possible (KISS), then please wrap in:
```
if (LOG.isDebugEnabled()) {
...
}
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] belugabehr commented on pull request #1265: [DRAFT] Support for HS2 client metadata cache
Posted by GitBox <gi...@apache.org>.
belugabehr commented on pull request #1265:
URL: https://github.com/apache/hive/pull/1265#issuecomment-659559921
For timing measurements, Hive has a 'PerfLogger' class
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] vineetgarg02 commented on a change in pull request #1265: [DRAFT] Support for HS2 client metadata cache
Posted by GitBox <gi...@apache.org>.
vineetgarg02 commented on a change in pull request #1265:
URL: https://github.com/apache/hive/pull/1265#discussion_r460140912
##########
File path: standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientWithLocalCache.java
##########
@@ -214,4 +214,13 @@ protected PartitionsSpecByExprResult getPartitionsSpecByExprResult(PartitionsByE
return r;
}
+
+ private boolean isRequestCachable(Object request) {
Review comment:
Add a comment explaining what sort of request we are trying to cache and avoid caching.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] belugabehr commented on a change in pull request #1265: [DRAFT] Support for HS2 client metadata cache
Posted by GitBox <gi...@apache.org>.
belugabehr commented on a change in pull request #1265:
URL: https://github.com/apache/hive/pull/1265#discussion_r455956580
##########
File path: standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientWithLocalCache.java
##########
@@ -0,0 +1,179 @@
+package org.apache.hadoop.hive.metastore;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.PartitionsByExprRequest;
+import org.apache.hadoop.hive.metastore.api.PartitionsByExprResult;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.ql.util.IncrementalObjectSizeEstimator;
+import org.apache.hadoop.hive.ql.util.IncrementalObjectSizeEstimator.ObjectEstimator;
+import org.apache.thrift.TException;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Objects;
+
+public class HiveMetaStoreClientWithLocalCache extends HiveMetaStoreClient {
+
+ private static LoadingCache<CacheKey, Object> mscLocalCache;
+ private static volatile boolean cacheInitialized = false;
+ private boolean isCacheEnabled = true;
+ private static HashMap<Class<?>, ObjectEstimator> sizeEstimator = null;
+
+ public HiveMetaStoreClientWithLocalCache(Configuration conf) throws MetaException {
+ this(conf, null, true);
+ }
+
+ public HiveMetaStoreClientWithLocalCache(Configuration conf, HiveMetaHookLoader hookLoader) throws MetaException {
+ this(conf, hookLoader, true);
+ }
+
+ public HiveMetaStoreClientWithLocalCache(Configuration conf, HiveMetaHookLoader hookLoader, Boolean allowEmbedded) throws MetaException {
+ super(conf, hookLoader, allowEmbedded);
+
+// isCacheEnabled = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.MSC_CACHE_ENABLED);
+ if (isCacheEnabled) {
+ // if cache is not initialized, init size estimator and the cache.
+ if (!cacheInitialized) {
+ cacheInitialized = true;
+ LOG.debug("Initializing local cache in HiveMetaStoreClient...");
+ // initialize a size estimator with this class and all other classes in KeyType.
+ initSizeEstimator();
+ initCache();
+ LOG.debug("Local cache initialized in HiveMetaStoreClient: " + mscLocalCache);
+ }
+ }
+ }
+
+ private static void initSizeEstimator() {
+ sizeEstimator = IncrementalObjectSizeEstimator.createEstimators(HiveMetaStoreClientWithLocalCache.class);
+ Arrays.stream(KeyType.values()).forEach(e -> {
+ IncrementalObjectSizeEstimator.createEstimators(e.keyClass, sizeEstimator);
+ IncrementalObjectSizeEstimator.createEstimators(e.valueClass, sizeEstimator);}
+ );
+ }
+
+ public enum KeyType {
+ PARTITIONS_BY_EXPR_REQUEST(PartitionsByExprRequest.class, PartitionsByExprResult.class);
+
+ private final Class<?> keyClass;
+ private final Class<?> valueClass;
+
+ KeyType(Class<?> keyClass, Class<?> valueClass) {
+ this.keyClass = keyClass;
+ this.valueClass = valueClass;
+ }
+ }
+
+ public static class CacheKey{
+ KeyType IDENTIFIER;
+ Object obj;
+
+ public CacheKey(KeyType IDENTIFIER, Object obj) {
+ this.IDENTIFIER = IDENTIFIER;
+ this.obj = obj;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ CacheKey cacheKey = (CacheKey) o;
+ return IDENTIFIER == cacheKey.IDENTIFIER &&
+ Objects.equals(obj, cacheKey.obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(IDENTIFIER, obj);
+ }
+ }
+
+ private static int getWeight(CacheKey key, Object val) {
+ if (val instanceof Exception) return 0;
+ ObjectEstimator keySizeEstimator = sizeEstimator.get(key.IDENTIFIER.keyClass);
+ ObjectEstimator valSizeEstimator = sizeEstimator.get(key.IDENTIFIER.valueClass);
+ int keySize = keySizeEstimator.estimate(key, sizeEstimator);
+ int valSize = valSizeEstimator.estimate(val, sizeEstimator);
+ LOG.debug("Cache entry weight - key: {}, value: {}, total: {}", keySize, valSize, keySize+valSize);
+ return keySize + valSize;
+ }
+
+ private Object getOrLoad(CacheKey key) {
+ Object val;
+ try {
+ val = getResultObject(key);
+ } catch (Exception e) {
+ LOG.debug("Exception in MSC local cache: " + e.toString());
+ if (e instanceof MetaException) {
+ val = new MetaException(e.getMessage());
+ } else {
+ val = new Exception(e.getMessage());
+ }
+ }
+ return val;
+ }
+
+ private synchronized void initCache() {
+ int maxSize = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.MSC_CACHE_MAX_SIZE);
+ int initSize = 100;
+ mscLocalCache = Caffeine.newBuilder()
+ .initialCapacity(initSize)
+ .maximumWeight(maxSize)
+ .weigher(HiveMetaStoreClientWithLocalCache::getWeight)
+ .removalListener((key, val, cause) ->
+ LOG.debug(String.format("Caffeine - (%s, %s) was removed (%s)", key, val, cause)))
+ .recordStats()
+ .build(this::getOrLoad);
+ }
+
+
+ protected Object getResultObject(CacheKey cacheKey) throws TException {
+ Object result = null;
+
+ switch (cacheKey.IDENTIFIER) {
+ case PARTITIONS_BY_EXPR_REQUEST:
+ result = super.getPartitionsByExprResult((PartitionsByExprRequest)cacheKey.obj);
+ break;
+ default:
+ break;
+ }
+
+ return result;
+ }
+
+ @Override
+ protected PartitionsByExprResult getPartitionsByExprResult(PartitionsByExprRequest req) throws TException {
+ PartitionsByExprResult r = null;
+
+ if (isCacheEnabled) {
+ CacheKey cacheKey = new CacheKey(KeyType.PARTITIONS_BY_EXPR_REQUEST, req);
+ Object val;
+ try {
+ val = mscLocalCache.get(cacheKey); // get either the result or an Exception
+
+ if (val instanceof PartitionsByExprResult) {
+ r = (PartitionsByExprResult) val;
+ LOG.debug(mscLocalCache.toString().substring(mscLocalCache.toString().indexOf("LoadingCache")) + ": " +
+ mscLocalCache.stats().toString());
+ } else if (val instanceof Exception) {
+ mscLocalCache.invalidate(cacheKey);
+ throw (Exception)val;
+ }
+ } catch (Exception e) {
+ if (e instanceof MetaException) {
+ throw new MetaException(e.getMessage());
+ } else {
+ throw new TException(e.getMessage());
+ }
Review comment:
```
} catch (MetaException me) {
throw me;
} catch (Exception e) {
throw new TException(e.getMessage());
}
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor edited a comment on pull request #1265: [DRAFT] Support for HS2 client metadata cache
Posted by GitBox <gi...@apache.org>.
jcamachor edited a comment on pull request #1265:
URL: https://github.com/apache/hive/pull/1265#issuecomment-659655095
@belugabehr , thanks for your feedback. That is what we are studying right now. API is compatible with Guava; the idea is to have a first prototype and being able to evaluate that.
I also moved this to draft since it was not ready to be reviewed.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] soumyakanti3578 commented on pull request #1265: [DRAFT] Support for HS2 client metadata cache
Posted by GitBox <gi...@apache.org>.
soumyakanti3578 commented on pull request #1265:
URL: https://github.com/apache/hive/pull/1265#issuecomment-663717470
Closing PR. Not able to build.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] belugabehr commented on a change in pull request #1265: [DRAFT] Support for HS2 client metadata cache
Posted by GitBox <gi...@apache.org>.
belugabehr commented on a change in pull request #1265:
URL: https://github.com/apache/hive/pull/1265#discussion_r455952798
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
##########
@@ -1631,6 +1633,9 @@ public Table apply(org.apache.hadoop.hive.metastore.api.Table table) {
}
} catch (Exception e) {
throw new HiveException(e);
+ } finally {
+ long diff = System.nanoTime() - t1;
+ LOG.debug(String.format(logString, "getTablesByType", diff, dbName, ""));
Review comment:
Please use SLF4J format here
```
LOG.debug("HS2 {} call took {} nano seconds - dbName: {}, tblName: {}", "getTablesByType", diff, dbName, "");
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] belugabehr commented on a change in pull request #1265: [DRAFT] Support for HS2 client metadata cache
Posted by GitBox <gi...@apache.org>.
belugabehr commented on a change in pull request #1265:
URL: https://github.com/apache/hive/pull/1265#discussion_r455954924
##########
File path: standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientWithLocalCache.java
##########
@@ -0,0 +1,179 @@
+package org.apache.hadoop.hive.metastore;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.PartitionsByExprRequest;
+import org.apache.hadoop.hive.metastore.api.PartitionsByExprResult;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.ql.util.IncrementalObjectSizeEstimator;
+import org.apache.hadoop.hive.ql.util.IncrementalObjectSizeEstimator.ObjectEstimator;
+import org.apache.thrift.TException;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Objects;
+
+public class HiveMetaStoreClientWithLocalCache extends HiveMetaStoreClient {
+
+ private static LoadingCache<CacheKey, Object> mscLocalCache;
+ private static volatile boolean cacheInitialized = false;
+ private boolean isCacheEnabled = true;
+ private static HashMap<Class<?>, ObjectEstimator> sizeEstimator = null;
+
+ public HiveMetaStoreClientWithLocalCache(Configuration conf) throws MetaException {
+ this(conf, null, true);
+ }
+
+ public HiveMetaStoreClientWithLocalCache(Configuration conf, HiveMetaHookLoader hookLoader) throws MetaException {
+ this(conf, hookLoader, true);
+ }
+
+ public HiveMetaStoreClientWithLocalCache(Configuration conf, HiveMetaHookLoader hookLoader, Boolean allowEmbedded) throws MetaException {
+ super(conf, hookLoader, allowEmbedded);
+
+// isCacheEnabled = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.MSC_CACHE_ENABLED);
+ if (isCacheEnabled) {
+ // if cache is not initialized, init size estimator and the cache.
+ if (!cacheInitialized) {
+ cacheInitialized = true;
+ LOG.debug("Initializing local cache in HiveMetaStoreClient...");
+ // initialize a size estimator with this class and all other classes in KeyType.
+ initSizeEstimator();
+ initCache();
+ LOG.debug("Local cache initialized in HiveMetaStoreClient: " + mscLocalCache);
+ }
+ }
+ }
+
+ private static void initSizeEstimator() {
+ sizeEstimator = IncrementalObjectSizeEstimator.createEstimators(HiveMetaStoreClientWithLocalCache.class);
+ Arrays.stream(KeyType.values()).forEach(e -> {
+ IncrementalObjectSizeEstimator.createEstimators(e.keyClass, sizeEstimator);
+ IncrementalObjectSizeEstimator.createEstimators(e.valueClass, sizeEstimator);}
+ );
+ }
+
+ public enum KeyType {
+ PARTITIONS_BY_EXPR_REQUEST(PartitionsByExprRequest.class, PartitionsByExprResult.class);
+
+ private final Class<?> keyClass;
+ private final Class<?> valueClass;
+
+ KeyType(Class<?> keyClass, Class<?> valueClass) {
+ this.keyClass = keyClass;
+ this.valueClass = valueClass;
+ }
+ }
+
+ public static class CacheKey{
+ KeyType IDENTIFIER;
+ Object obj;
+
+ public CacheKey(KeyType IDENTIFIER, Object obj) {
+ this.IDENTIFIER = IDENTIFIER;
+ this.obj = obj;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ CacheKey cacheKey = (CacheKey) o;
+ return IDENTIFIER == cacheKey.IDENTIFIER &&
+ Objects.equals(obj, cacheKey.obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(IDENTIFIER, obj);
+ }
+ }
+
+ private static int getWeight(CacheKey key, Object val) {
+ if (val instanceof Exception) return 0;
+ ObjectEstimator keySizeEstimator = sizeEstimator.get(key.IDENTIFIER.keyClass);
+ ObjectEstimator valSizeEstimator = sizeEstimator.get(key.IDENTIFIER.valueClass);
+ int keySize = keySizeEstimator.estimate(key, sizeEstimator);
+ int valSize = valSizeEstimator.estimate(val, sizeEstimator);
+ LOG.debug("Cache entry weight - key: {}, value: {}, total: {}", keySize, valSize, keySize+valSize);
+ return keySize + valSize;
+ }
+
+ private Object getOrLoad(CacheKey key) {
+ Object val;
+ try {
+ val = getResultObject(key);
+ } catch (Exception e) {
+ LOG.debug("Exception in MSC local cache: " + e.toString());
+ if (e instanceof MetaException) {
+ val = new MetaException(e.getMessage());
Review comment:
Why does this return an Exception? Exceptions are meant to be thrown. The calling class doesn't do anything with it anyway. Please throw.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] vineetgarg02 commented on a change in pull request #1265: [DRAFT] Support for HS2 client metadata cache
Posted by GitBox <gi...@apache.org>.
vineetgarg02 commented on a change in pull request #1265:
URL: https://github.com/apache/hive/pull/1265#discussion_r460141221
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
##########
@@ -4007,9 +4007,17 @@ public boolean getPartitionsByExpr(Table tbl, ExprNodeGenericFuncDesc expr, Hive
if (AcidUtils.isTransactionalTable(tbl)) {
validWriteIdList = getValidWriteIdList(tbl.getDbName(), tbl.getTableName());
}
- boolean hasUnknownParts = getMSC().listPartitionsSpecByExpr(tbl.getDbName(),
- tbl.getTableName(), exprBytes, defaultPartitionName, (short)-1, msParts,
- validWriteIdList != null ? validWriteIdList.toString() : null);
+
+ PartitionsByExprRequest req = new PartitionsByExprRequest(tbl.getDbName(), tbl.getTableName(),
Review comment:
It will be good to encapsulate creation of `PartitionsByExprRequest` is a separate method.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] belugabehr commented on pull request #1265: [DRAFT] Support for HS2 client metadata cache
Posted by GitBox <gi...@apache.org>.
belugabehr commented on pull request #1265:
URL: https://github.com/apache/hive/pull/1265#issuecomment-659564705
We already have Guava dependency, why introduce Caffeine? Is it really that much better in this use case?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] belugabehr commented on a change in pull request #1265: [DRAFT] Support for HS2 client metadata cache
Posted by GitBox <gi...@apache.org>.
belugabehr commented on a change in pull request #1265:
URL: https://github.com/apache/hive/pull/1265#discussion_r455958403
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
##########
@@ -3665,36 +3678,42 @@ public boolean dropPartition(String dbName, String tableName, List<String> parti
* @return list of partition objects
*/
public List<Partition> getPartitions(Table tbl) throws HiveException {
- if (tbl.isPartitioned()) {
- List<org.apache.hadoop.hive.metastore.api.Partition> tParts;
- try {
- GetPartitionsPsWithAuthRequest req = new GetPartitionsPsWithAuthRequest();
- req.setTblName(tbl.getTableName());
- req.setDbName(tbl.getDbName());
- req.setUserName(getUserName());
- req.setMaxParts((short) -1);
- req.setGroupNames(getGroupNames());
- if (AcidUtils.isTransactionalTable(tbl)) {
- ValidWriteIdList validWriteIdList = getValidWriteIdList(tbl.getDbName(), tbl.getTableName());
- req.setValidWriteIdList(validWriteIdList != null ? validWriteIdList.toString() : null);
- }
- GetPartitionsPsWithAuthResponse res = getMSC().listPartitionsWithAuthInfoRequest(req);
- tParts = res.getPartitions();
+ long t1 = System.nanoTime();
+ try {
+ if (tbl.isPartitioned()) {
+ List<org.apache.hadoop.hive.metastore.api.Partition> tParts;
+ try {
+ GetPartitionsPsWithAuthRequest req = new GetPartitionsPsWithAuthRequest();
+ req.setTblName(tbl.getTableName());
+ req.setDbName(tbl.getDbName());
+ req.setUserName(getUserName());
+ req.setMaxParts((short) -1);
+ req.setGroupNames(getGroupNames());
+ if (AcidUtils.isTransactionalTable(tbl)) {
+ ValidWriteIdList validWriteIdList = getValidWriteIdList(tbl.getDbName(), tbl.getTableName());
+ req.setValidWriteIdList(validWriteIdList != null ? validWriteIdList.toString() : null);
+ }
+ GetPartitionsPsWithAuthResponse res = getMSC().listPartitionsWithAuthInfoRequest(req);
+ tParts = res.getPartitions();
- } catch (Exception e) {
- LOG.error(StringUtils.stringifyException(e));
- throw new HiveException(e);
- }
- List<Partition> parts = new ArrayList<Partition>(tParts.size());
- for (org.apache.hadoop.hive.metastore.api.Partition tpart : tParts) {
- parts.add(new Partition(tbl, tpart));
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ throw new HiveException(e);
+ }
+ List<Partition> parts = new ArrayList<Partition>(tParts.size());
+ for (org.apache.hadoop.hive.metastore.api.Partition tpart : tParts) {
+ parts.add(new Partition(tbl, tpart));
+ }
+ return parts;
+ } else {
+ Partition part = new Partition(tbl);
+ ArrayList<Partition> parts = new ArrayList<Partition>(1);
Review comment:
Use:
```
Collections.singletonList
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on pull request #1265: [DRAFT] Support for HS2 client metadata cache
Posted by GitBox <gi...@apache.org>.
jcamachor commented on pull request #1265:
URL: https://github.com/apache/hive/pull/1265#issuecomment-659655095
@belugabehr , that is what we are studying right now. API is compatible with Guava; the idea is to have a first prototype and being able to evaluate that.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org