You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2018/05/08 18:52:33 UTC
hive git commit: HIVE-19446: QueryCache: Transaction lists needed for
pending cache entries (Jason Dere, reviewed by GopalV)
Repository: hive
Updated Branches:
refs/heads/master 3f86e4df4 -> 1121ab46e
HIVE-19446: QueryCache: Transaction lists needed for pending cache entries (Jason Dere, reviewed by GopalV)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1121ab46
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1121ab46
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1121ab46
Branch: refs/heads/master
Commit: 1121ab46ef4594c260312a813966637c5bff91c3
Parents: 3f86e4d
Author: Jason Dere <jd...@hortonworks.com>
Authored: Tue May 8 11:52:01 2018 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Tue May 8 11:52:01 2018 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 9 ++++++---
.../hive/ql/cache/results/QueryResultsCache.java | 14 +++++++++++---
2 files changed, 17 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1121ab46/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 9f4e6f2..52799b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -1977,10 +1977,14 @@ public class Driver implements IDriver {
if (cacheUsage != null) {
if (cacheUsage.getStatus() == CacheUsage.CacheStatus.CAN_CACHE_QUERY_RESULTS &&
plan.getFetchTask() != null) {
+ ValidTxnWriteIdList txnWriteIdList = null;
+ if (plan.hasAcidResourcesInQuery()) {
+ txnWriteIdList = AcidUtils.getValidTxnWriteIdList(conf);
+ }
// The results of this query execution might be cacheable.
// Add a placeholder entry in the cache so other queries know this result is pending.
CacheEntry pendingCacheEntry =
- QueryResultsCache.getInstance().addToCache(cacheUsage.getQueryInfo());
+ QueryResultsCache.getInstance().addToCache(cacheUsage.getQueryInfo(), txnWriteIdList);
if (pendingCacheEntry != null) {
// Update cacheUsage to reference the pending entry.
this.cacheUsage.setCacheEntry(pendingCacheEntry);
@@ -2011,8 +2015,7 @@ public class Driver implements IDriver {
}
boolean savedToCache = QueryResultsCache.getInstance().setEntryValid(
cacheUsage.getCacheEntry(),
- plan.getFetchTask().getWork(),
- txnWriteIdList);
+ plan.getFetchTask().getWork());
LOG.info("savedToCache: {}", savedToCache);
if (savedToCache) {
useFetchFromCache(cacheUsage.getCacheEntry());
http://git-wip-us.apache.org/repos/asf/hive/blob/1121ab46/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
index 6734ac5..11b9c27 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
@@ -488,11 +488,12 @@ public final class QueryResultsCache {
* @param queryInfo
* @return
*/
- public CacheEntry addToCache(QueryInfo queryInfo) {
+ public CacheEntry addToCache(QueryInfo queryInfo, ValidTxnWriteIdList txnWriteIdList) {
// Create placeholder entry with PENDING state.
String queryText = queryInfo.getLookupInfo().getQueryText();
CacheEntry addedEntry = new CacheEntry();
addedEntry.queryInfo = queryInfo;
+ addedEntry.txnWriteIdList = txnWriteIdList;
Lock writeLock = rwLock.writeLock();
try {
@@ -522,7 +523,7 @@ public final class QueryResultsCache {
* @param fetchWork
* @return
*/
- public boolean setEntryValid(CacheEntry cacheEntry, FetchWork fetchWork, ValidTxnWriteIdList txnWriteIdList) {
+ public boolean setEntryValid(CacheEntry cacheEntry, FetchWork fetchWork) {
String queryText = cacheEntry.getQueryText();
boolean dataDirMoved = false;
Path queryResultsPath = null;
@@ -571,7 +572,6 @@ public final class QueryResultsCache {
cacheEntry.cachedResultsPath = cachedResultsPath;
cacheEntry.size = resultSize;
this.cacheSize += resultSize;
- cacheEntry.txnWriteIdList = txnWriteIdList;
cacheEntry.setStatus(CacheEntryStatus.VALID);
// Mark this entry as being in use. Caller will need to release later.
@@ -700,6 +700,14 @@ public final class QueryResultsCache {
boolean writeIdCheckPassed = false;
String tableName = tableUsed.getFullyQualifiedName();
ValidTxnWriteIdList currentTxnWriteIdList = lookupInfo.txnWriteIdListProvider.get();
+ if (currentTxnWriteIdList == null) {
+ LOG.warn("Current query's txnWriteIdList is null!");
+ return false;
+ }
+ if (entry.txnWriteIdList == null) {
+ LOG.warn("Cache entry's txnWriteIdList is null!");
+ return false;
+ }
ValidWriteIdList currentWriteIdForTable =
currentTxnWriteIdList.getTableValidWriteIdList(tableName);
ValidWriteIdList cachedWriteIdForTable = entry.txnWriteIdList.getTableValidWriteIdList(tableName);