You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/05/07 20:00:26 UTC
[1/2] hive git commit: HIVE-19410 : don't create serde reader in LLAP
if there's no cache (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master 4a2bfb8be -> 0fd74db20
HIVE-19410 : don't create serde reader in LLAP if there's no cache (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c52e1955
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c52e1955
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c52e1955
Branch: refs/heads/master
Commit: c52e19553762ab34f45189cf0e70dfe7a9c0801c
Parents: 4a2bfb8
Author: sergey <se...@apache.org>
Authored: Mon May 7 12:09:04 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Mon May 7 12:11:40 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java | 3 ++-
.../hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java | 1 +
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c52e1955/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
index 747b399..5003d9b 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
@@ -133,7 +133,8 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
MetadataCache metadataCache = null;
SerDeLowLevelCacheImpl serdeCache = null; // TODO: extract interface when needed
BufferUsageManager bufferManagerOrc = null, bufferManagerGeneric = null;
- boolean isEncodeEnabled = HiveConf.getBoolVar(conf, ConfVars.LLAP_IO_ENCODE_ENABLED);
+ boolean isEncodeEnabled = useLowLevelCache
+ && HiveConf.getBoolVar(conf, ConfVars.LLAP_IO_ENCODE_ENABLED);
if (useLowLevelCache) {
// Memory manager uses cache policy to trigger evictions, so create the policy first.
boolean useLrfu = HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_USE_LRFU);
http://git-wip-us.apache.org/repos/asf/hive/blob/c52e1955/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
index 8b89ae9..1cfe929 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
@@ -178,6 +178,7 @@ public class SerDeEncodedDataReader extends CallableWithNdc<Void>
InputFormat<?, ?> sourceInputFormat, Deserializer sourceSerDe,
QueryFragmentCounters counters, TypeDescription schema, Map<Path, PartitionDesc> parts)
throws IOException {
+ assert cache != null;
this.cache = cache;
this.bufferManager = bufferManager;
this.bufferFactory = new BufferObjectFactory() {
[2/2] hive git commit: HIVE-19310 : Metastore:
MetaStoreDirectSql.ensureDbInit has some slow DN calls which might need to be
run only in test env (Sergey Shelukhin, reviewed by Gopal Vijayaraghavan)
Posted by se...@apache.org.
HIVE-19310 : Metastore: MetaStoreDirectSql.ensureDbInit has some slow DN calls which might need to be run only in test env (Sergey Shelukhin, reviewed by Gopal Vijayaraghavan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0fd74db2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0fd74db2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0fd74db2
Branch: refs/heads/master
Commit: 0fd74db2085c592b2ac620bf602e739589d39511
Parents: c52e195
Author: sergey <se...@apache.org>
Authored: Mon May 7 12:10:49 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Mon May 7 12:11:42 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0fd74db2/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 4e0e887..56fbfed 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -179,7 +179,8 @@ class MetaStoreDirectSql {
+ " Disabling directSQL as it uses hand-hardcoded SQL with that assumption.");
isCompatibleDatastore = false;
} else {
- isCompatibleDatastore = ensureDbInit() && runTestQuery();
+ boolean isInTest = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST);
+ isCompatibleDatastore = (!isInTest || ensureDbInit()) && runTestQuery();
if (isCompatibleDatastore) {
LOG.info("Using direct SQL, underlying DB is " + dbType);
}