You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ve...@apache.org on 2023/03/08 08:57:27 UTC
[hive] branch master updated: HIVE-27122: Use Caffeine for caching metadata objects in Compactor threads (Sourabh Badhya, reviewed by Laszlo Vegh, Akshat Matur)
This is an automated email from the ASF dual-hosted git repository.
veghlaci05 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 14c03791777 HIVE-27122: Use Caffeine for caching metadata objects in Compactor threads (Sourabh Badhya, reviewed by Laszlo Vegh, Akshat Matur)
14c03791777 is described below
commit 14c03791777b780cbef1533956874869b4ce5a58
Author: Sourabh Badhya <42...@users.noreply.github.com>
AuthorDate: Wed Mar 8 14:27:16 2023 +0530
HIVE-27122: Use Caffeine for caching metadata objects in Compactor threads (Sourabh Badhya, reviewed by Laszlo Vegh, Akshat Matur)
---
.../hive/ql/txn/compactor/CompactorUtil.java | 21 ++++++++++++++++++++
.../hadoop/hive/ql/txn/compactor/FSRemover.java | 13 ++++++++----
.../hadoop/hive/ql/txn/compactor/Initiator.java | 11 ++---------
.../ql/txn/compactor/MetaStoreCompactorThread.java | 7 +------
.../hive/ql/txn/compactor/MetadataCache.java | 23 +++++++++++-----------
.../hive/ql/txn/compactor/handler/TaskHandler.java | 10 +---------
6 files changed, 46 insertions(+), 39 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorUtil.java
index ddc6a0312aa..a8b45b0baec 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorUtil.java
@@ -20,11 +20,14 @@ package org.apache.hadoop.hive.ql.txn.compactor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.utils.StringableMap;
import org.apache.hadoop.hive.ql.io.AcidDirectory;
import org.slf4j.Logger;
@@ -133,6 +136,24 @@ public class CompactorUtil {
}
}
+ public static Database resolveDatabase(HiveConf conf, String dbName) throws MetaException, NoSuchObjectException {
+ try {
+ return getMSForConf(conf).getDatabase(MetaStoreUtils.getDefaultCatalog(conf), dbName);
+ } catch (NoSuchObjectException e) {
+ LOG.error("Unable to find database {}, {}", dbName, e.getMessage());
+ throw e;
+ }
+ }
+
+ public static Table resolveTable(HiveConf conf, String dbName, String tableName) throws MetaException {
+ try {
+ return getMSForConf(conf).getTable(MetaStoreUtils.getDefaultCatalog(conf), dbName, tableName);
+ } catch (MetaException e) {
+ LOG.error("Unable to find table {}.{}, {}", dbName, tableName, e.getMessage());
+ throw e;
+ }
+ }
+
public static String getDebugInfo(List<Path> paths) {
return "[" + paths.stream().map(Path::getName).collect(Collectors.joining(",")) + ']';
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/FSRemover.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/FSRemover.java
index e59a5490206..cffef09f127 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/FSRemover.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/FSRemover.java
@@ -37,9 +37,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
-import static org.apache.hadoop.hive.metastore.HMSHandler.getMSForConf;
-import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
-
/**
* A runnable class which takes in cleaningRequestHandler and cleaning request and deletes the files
* according to the cleaning request.
@@ -103,11 +100,19 @@ public class FSRemover {
cr.getLocation(), CompactorUtil.getDebugInfo(cr.getObsoleteDirs()));
boolean needCmRecycle;
try {
- Database db = metadataCache.computeIfAbsent(cr.getDbName(), () -> getMSForConf(conf).getDatabase(getDefaultCatalog(conf), cr.getDbName()));
+ Database db = metadataCache.computeIfAbsent(cr.getDbName(),
+ () -> CompactorUtil.resolveDatabase(conf, cr.getDbName()));
needCmRecycle = ReplChangeManager.isSourceOfReplication(db);
} catch (NoSuchObjectException ex) {
// can not drop a database which is a source of replication
needCmRecycle = false;
+ } catch (RuntimeException ex) {
+ if (ex.getCause() instanceof NoSuchObjectException) {
+ // can not drop a database which is a source of replication
+ needCmRecycle = false;
+ } else {
+ throw ex;
+ }
} catch (Exception ex) {
throw new MetaException(ex.getMessage());
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
index 284a6ffca15..9398258a64b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
@@ -81,7 +81,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import static org.apache.hadoop.hive.conf.Constants.COMPACTOR_INTIATOR_THREAD_NAME_FORMAT;
-import static org.apache.hadoop.hive.metastore.HMSHandler.getMSForConf;
/**
* A class to initiate compactions. This will run in a separate thread.
@@ -179,7 +178,7 @@ public class Initiator extends MetaStoreCompactorThread {
return;
}
- Table t = metadataCache.computeIfAbsent(ci.getFullTableName(),() -> resolveTable(ci));
+ Table t = metadataCache.computeIfAbsent(ci.getFullTableName(), () -> resolveTable(ci));
String poolName = getPoolName(ci, t);
Partition p = resolvePartition(ci);
if (p == null && ci.partName != null) {
@@ -292,14 +291,8 @@ public class Initiator extends MetaStoreCompactorThread {
return poolName;
}
-
private Database resolveDatabase(CompactionInfo ci) throws MetaException, NoSuchObjectException {
- try {
- return getMSForConf(conf).getDatabase(MetaStoreUtils.getDefaultCatalog(conf), ci.dbname);
- } catch (NoSuchObjectException e) {
- LOG.error("Unable to find database " + ci.dbname + ", " + e.getMessage());
- throw e;
- }
+ return CompactorUtil.resolveDatabase(conf, ci.dbname);
}
private ValidWriteIdList resolveValidWriteIds(Table t) throws NoSuchTxnException, MetaException {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetaStoreCompactorThread.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetaStoreCompactorThread.java
index d7871b47054..7cda085ff6b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetaStoreCompactorThread.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetaStoreCompactorThread.java
@@ -66,12 +66,7 @@ public abstract class MetaStoreCompactorThread extends CompactorThread implement
}
@Override Table resolveTable(CompactionInfo ci) throws MetaException {
- try {
- return getMSForConf(conf).getTable(getDefaultCatalog(conf), ci.dbname, ci.tableName);
- } catch (MetaException e) {
- LOG.error("Unable to find table " + ci.getFullTableName() + ", " + e.getMessage());
- throw e;
- }
+ return CompactorUtil.resolveTable(conf, ci.dbname, ci.tableName);
}
@Override boolean replIsCompactionDisabledForDatabase(String dbName) throws TException {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetadataCache.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetadataCache.java
index a09491b4e2f..d3c7e1d64ad 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetadataCache.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetadataCache.java
@@ -17,30 +17,31 @@
*/
package org.apache.hadoop.hive.ql.txn.compactor;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
import org.apache.thrift.TBase;
import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
public class MetadataCache {
private Cache<String, TBase> metaCache;
- public MetadataCache(boolean tableCacheOn) {
- if (tableCacheOn) {
- metaCache = CacheBuilder.newBuilder().softValues().build();
+ public MetadataCache(boolean isCacheEnabled) {
+ if (isCacheEnabled) {
+ metaCache = Caffeine.newBuilder().softValues().build();
}
}
public <T extends TBase<T,?>> T computeIfAbsent(String key, Callable<T> callable) throws Exception {
if (metaCache != null) {
- try {
- return (T) metaCache.get(key, callable);
- } catch (ExecutionException e) {
- throw (Exception) e.getCause();
- }
+ return (T) metaCache.get(key, k -> {
+ try {
+ return callable.call();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
}
return callable.call();
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/TaskHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/TaskHandler.java
index d804a318e54..327b9238486 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/TaskHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/TaskHandler.java
@@ -31,9 +31,6 @@ import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.List;
-import static org.apache.hadoop.hive.metastore.HMSHandler.getMSForConf;
-import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
-
/**
* An abstract class which defines the list of utility methods for performing cleanup activities.
*/
@@ -58,12 +55,7 @@ public abstract class TaskHandler {
public abstract List<Runnable> getTasks() throws MetaException;
protected Table resolveTable(String dbName, String tableName) throws MetaException {
- try {
- return getMSForConf(conf).getTable(getDefaultCatalog(conf), dbName, tableName);
- } catch (MetaException e) {
- LOG.error("Unable to find table {}.{}, {}", dbName, tableName, e.getMessage());
- throw e;
- }
+ return CompactorUtil.resolveTable(conf, dbName, tableName);
}
protected Partition resolvePartition(String dbName, String tableName, String partName) throws MetaException {