You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2019/04/23 19:48:37 UTC
[hive] branch master updated: Revert "HIVE-20615: CachedStore:
Background refresh thread bug fixes (Vaibhav Gumashta,
reviewed by Daniel Dai)"
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 313eb24 Revert "HIVE-20615: CachedStore: Background refresh thread bug fixes (Vaibhav Gumashta, reviewed by Daniel Dai)"
313eb24 is described below
commit 313eb24d054116d1b020127017aa11aae906deaf
Author: Jesus Camacho Rodriguez <jc...@apache.org>
AuthorDate: Tue Apr 23 12:48:26 2019 -0700
Revert "HIVE-20615: CachedStore: Background refresh thread bug fixes (Vaibhav Gumashta, reviewed by Daniel Dai)"
This reverts commit b58d50cb73a1f79a5d079e0a2c5ac33d2efc33a0.
---
.../hadoop/hive/metastore/cache/CacheUtils.java | 4 ---
.../hadoop/hive/metastore/cache/CachedStore.java | 4 +--
.../hadoop/hive/metastore/cache/SharedCache.java | 29 ++++++++--------------
3 files changed, 12 insertions(+), 25 deletions(-)
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java
index d50fa13..944c813 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java
@@ -40,10 +40,6 @@ public class CacheUtils {
return buildKey(catName.toLowerCase(), dbName.toLowerCase());
}
- public static String buildDbKeyWithDelimiterSuffix(String catName, String dbName) {
- return buildKey(catName.toLowerCase(), dbName.toLowerCase()) + delimit;
- }
-
/**
* Builds a key for the partition cache which is concatenation of partition values, each value
* separated by a delimiter
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index 6ef9a19..e366ebd 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -724,7 +724,6 @@ public class CachedStore implements RawStore, Configurable {
} else {
try {
triggerPreWarm(rawStore);
- shouldRunPrewarm = false;
} catch (Exception e) {
LOG.error("Prewarm failure", e);
return;
@@ -816,6 +815,7 @@ public class CachedStore implements RawStore, Configurable {
if (table != null && !table.isSetPartitionKeys()) {
List<String> colNames = MetaStoreUtils.getColumnNamesForTable(table);
Deadline.startTimer("getTableColumnStatistics");
+
ColumnStatistics tableColStats =
rawStore.getTableColumnStatistics(catName, dbName, tblName, colNames);
Deadline.stopTimer();
@@ -865,9 +865,7 @@ public class CachedStore implements RawStore, Configurable {
rawStore.getPartitionColumnStatistics(catName, dbName, tblName, partNames, colNames);
Deadline.stopTimer();
sharedCache.refreshPartitionColStatsInCache(catName, dbName, tblName, partitionColStats);
- Deadline.startTimer("getPartitionsByNames");
List<Partition> parts = rawStore.getPartitionsByNames(catName, dbName, tblName, partNames);
- Deadline.stopTimer();
// Also save partitions for consistency as they have the stats state.
for (Partition part : parts) {
sharedCache.alterPartitionInCache(catName, dbName, tblName, part.getValues(), part);
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
index 60862d4..1c23022 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
@@ -1317,13 +1317,11 @@ public class SharedCache {
//in case of retry, ignore second try.
return;
}
- if (tblWrapper != null) {
- byte[] sdHash = tblWrapper.getSdHash();
- if (sdHash != null) {
- decrSd(sdHash);
- }
- isTableCacheDirty.set(true);
+ byte[] sdHash = tblWrapper.getSdHash();
+ if (sdHash != null) {
+ decrSd(sdHash);
}
+ isTableCacheDirty.set(true);
} finally {
cacheLock.writeLock().unlock();
}
@@ -1440,30 +1438,25 @@ public class SharedCache {
public void refreshTablesInCache(String catName, String dbName, List<Table> tables) {
try {
+ cacheLock.writeLock().lock();
if (isTableCacheDirty.compareAndSet(true, false)) {
LOG.debug("Skipping table cache update; the table list we have is dirty.");
return;
}
- Map<String, TableWrapper> newCacheForDB = new TreeMap<>();
+ Map<String, TableWrapper> newTableCache = new HashMap<>();
for (Table tbl : tables) {
String tblName = StringUtils.normalizeIdentifier(tbl.getTableName());
- TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName));
+ TableWrapper tblWrapper =
+ tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName));
if (tblWrapper != null) {
tblWrapper.updateTableObj(tbl, this);
} else {
tblWrapper = createTableWrapper(catName, dbName, tblName, tbl);
}
- newCacheForDB.put(CacheUtils.buildTableKey(catName, dbName, tblName), tblWrapper);
- }
- cacheLock.writeLock().lock();
- Iterator<Entry<String, TableWrapper>> entryIterator = tableCache.entrySet().iterator();
- while (entryIterator.hasNext()) {
- String key = entryIterator.next().getKey();
- if (key.startsWith(CacheUtils.buildDbKeyWithDelimiterSuffix(catName, dbName))) {
- entryIterator.remove();
- }
+ newTableCache.put(CacheUtils.buildTableKey(catName, dbName, tblName), tblWrapper);
}
- tableCache.putAll(newCacheForDB);
+ tableCache.clear();
+ tableCache = newTableCache;
} finally {
cacheLock.writeLock().unlock();
}