You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kl...@apache.org on 2020/09/28 08:21:32 UTC
[hive] branch master updated: HIVE-24168: Disable
hdfsEncryptionShims cache during query-based compaction (Karen Coppage,
reviewed by Adam Szita)
This is an automated email from the ASF dual-hosted git repository.
klcopp pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new a8d12dd HIVE-24168: Disable hdfsEncryptionShims cache during query-based compaction (Karen Coppage, reviewed by Adam Szita)
a8d12dd is described below
commit a8d12dd7eeb980fd4bb0ec1c9933f93f5a8e058a
Author: Karen Coppage <ka...@gmail.com>
AuthorDate: Mon Sep 28 10:21:18 2020 +0200
HIVE-24168: Disable hdfsEncryptionShims cache during query-based compaction (Karen Coppage, reviewed by Adam Szita)
Closes #1501.
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 6 +++-
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 4 +--
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 3 +-
.../hadoop/hive/ql/session/SessionState.java | 35 ++++++++++++++--------
.../hive/ql/txn/compactor/QueryCompactor.java | 1 +
5 files changed, 33 insertions(+), 16 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 04325aa..1e6e5ca 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4050,7 +4050,11 @@ public class HiveConf extends Configuration {
" are produced by this query and finish earlier will be available for querying\n" +
" much earlier. Since the locks are only released once the query finishes, this\n" +
" does not apply if concurrency is enabled."),
-
+ HIVE_HDFS_ENCRYPTION_SHIM_CACHE_ON("hive.hdfs.encryption.shim.cache.on", true,
+ "Hive keeps a cache of hdfs encryption shims in SessionState. Each encryption shim in the cache stores a "
+ + "FileSystem object. If one of these FileSystems is closed anywhere in the system and HDFS config"
+ + "fs.hdfs.impl.disable.cache is false, its encryption shim in the cache will be unusable. "
+ + "If this is config set to false, then the encryption shim cache will be disabled."),
HIVE_INFER_BUCKET_SORT("hive.exec.infer.bucket.sort", false,
"If this is set, when writing partitions, the metadata will include the bucketing/sorting\n" +
"properties with which the data was written if any (this will not overwrite the metadata\n" +
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index d898e44..f6a9b34 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -4649,8 +4649,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
return false;
}
//Check if different encryption zones
- HadoopShims.HdfsEncryptionShim srcHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(srcFs);
- HadoopShims.HdfsEncryptionShim destHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(destFs);
+ HadoopShims.HdfsEncryptionShim srcHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(srcFs, conf);
+ HadoopShims.HdfsEncryptionShim destHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(destFs, conf);
try {
return srcHdfsEncryptionShim != null
&& destHdfsEncryptionShim != null
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index b5a752e..d0026a7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -2507,7 +2507,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
private boolean isPathEncrypted(Path path) throws HiveException {
try {
- HadoopShims.HdfsEncryptionShim hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(path.getFileSystem(conf));
+ HadoopShims.HdfsEncryptionShim hdfsEncryptionShim =
+ SessionState.get().getHdfsEncryptionShim(path.getFileSystem(conf), conf);
if (hdfsEncryptionShim != null) {
if (hdfsEncryptionShim.isPathEncrypted(path)) {
return true;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 4bcf602..1c3537f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -549,7 +549,7 @@ public class SessionState implements ISessionAuthState{
}
public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim() throws HiveException {
try {
- return getHdfsEncryptionShim(FileSystem.get(sessionConf));
+ return getHdfsEncryptionShim(FileSystem.get(sessionConf), sessionConf);
}
catch(HiveException hiveException) {
throw hiveException;
@@ -559,20 +559,31 @@ public class SessionState implements ISessionAuthState{
}
}
- public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim(FileSystem fs) throws HiveException {
- if (!hdfsEncryptionShims.containsKey(fs.getUri())) {
- try {
- if ("hdfs".equals(fs.getUri().getScheme())) {
- hdfsEncryptionShims.put(fs.getUri(), ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, sessionConf));
- } else {
- LOG.info("Could not get hdfsEncryptionShim, it is only applicable to hdfs filesystem.");
- }
- } catch (Exception e) {
- throw new HiveException(e);
+ public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim(FileSystem fs, HiveConf conf) throws HiveException {
+
+ if (!"hdfs".equals(fs.getUri().getScheme())) {
+ LOG.warn("Unable to get hdfs encryption shim, because FileSystem URI schema is not hdfs. Returning null. "
+ + "FileSystem URI: " + fs.getUri());
+ return null;
+ }
+
+ if (conf.getBoolVar(ConfVars.HIVE_HDFS_ENCRYPTION_SHIM_CACHE_ON)) {
+ if (!hdfsEncryptionShims.containsKey(fs.getUri())) {
+ hdfsEncryptionShims.put(fs.getUri(), getHdfsEncryptionShimInternal(fs));
}
+ return hdfsEncryptionShims.get(fs.getUri());
+
+ } else { // skip the cache
+ return getHdfsEncryptionShimInternal(fs);
}
+ }
- return hdfsEncryptionShims.get(fs.getUri());
+ private HadoopShims.HdfsEncryptionShim getHdfsEncryptionShimInternal(FileSystem fs) throws HiveException {
+ try {
+ return ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, sessionConf);
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
}
// SessionState is not available in runtime and Hive.get().getConf() is not safe to call
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
index 29535a2..ba3513d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
@@ -95,6 +95,7 @@ abstract class QueryCompactor {
throws IOException {
Util.disableLlapCaching(conf);
conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, true);
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_HDFS_ENCRYPTION_SHIM_CACHE_ON, false);
String user = compactionInfo.runAs;
SessionState sessionState = DriverUtils.setUpSessionState(conf, user, true);
long compactorTxnId = CompactorMR.CompactorMap.getCompactorTxnId(conf);