You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2016/08/03 15:58:51 UTC
hive git commit: HIVE-14380: Queries on tables with remote HDFS paths
fail in "encryption" checks. (Mithun Radhakrishnan, reviewed by Sergio Pena)
Repository: hive
Updated Branches:
refs/heads/master 7f43dcb39 -> 4817d8bd9
HIVE-14380: Queries on tables with remote HDFS paths fail in "encryption" checks. (Mithun Radhakrishnan, reviewed by Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4817d8bd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4817d8bd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4817d8bd
Branch: refs/heads/master
Commit: 4817d8bd9bbffcf836c940016c97ce8d8d316542
Parents: 7f43dcb
Author: Mithun Radhakrishnan <my...@gmail.com>
Authored: Wed Aug 3 10:57:42 2016 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed Aug 3 10:57:42 2016 -0500
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/metadata/Hive.java | 9 +++++---
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 11 ++++-----
.../hadoop/hive/ql/session/SessionState.java | 24 +++++++++++++++-----
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 21 +++++++++++++----
.../apache/hadoop/hive/shims/HadoopShims.java | 17 ++++++++++++++
5 files changed, 63 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4817d8bd/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 9a24ec3..57433bb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -3034,10 +3034,13 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
//Check if different encryption zones
- HadoopShims.HdfsEncryptionShim hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim();
+ HadoopShims.HdfsEncryptionShim srcHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(srcFs);
+ HadoopShims.HdfsEncryptionShim destHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(destFs);
try {
- return hdfsEncryptionShim != null && (hdfsEncryptionShim.isPathEncrypted(srcf) || hdfsEncryptionShim.isPathEncrypted(destf))
- && !hdfsEncryptionShim.arePathsOnSameEncryptionZone(srcf, destf);
+ return srcHdfsEncryptionShim != null
+ && destHdfsEncryptionShim != null
+ && (srcHdfsEncryptionShim.isPathEncrypted(srcf) || destHdfsEncryptionShim.isPathEncrypted(destf))
+ && !srcHdfsEncryptionShim.arePathsOnSameEncryptionZone(srcf, destf, destHdfsEncryptionShim);
} catch (IOException e) {
throw new HiveException(e);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4817d8bd/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 2671cb1..a01a7bd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -2196,17 +2196,16 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* @throws HiveException If an error occurs while checking for encryption
*/
private boolean isPathEncrypted(Path path) throws HiveException {
- HadoopShims.HdfsEncryptionShim hdfsEncryptionShim;
- hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim();
- if (hdfsEncryptionShim != null) {
- try {
+ try {
+ HadoopShims.HdfsEncryptionShim hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(path.getFileSystem(conf));
+ if (hdfsEncryptionShim != null) {
if (hdfsEncryptionShim.isPathEncrypted(path)) {
return true;
}
- } catch (Exception e) {
- throw new HiveException("Unable to determine if " + path + " is encrypted: " + e, e);
}
+ } catch (Exception e) {
+ throw new HiveException("Unable to determine if " + path + " is encrypted: " + e, e);
}
return false;
http://git-wip-us.apache.org/repos/asf/hive/blob/4817d8bd/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 83a8610..0581bab 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -42,6 +42,7 @@ import java.util.UUID;
import java.util.concurrent.CancellationException;
import java.util.concurrent.locks.ReentrantLock;
+import com.google.common.collect.Maps;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.slf4j.Logger;
@@ -211,7 +212,7 @@ public class SessionState {
/**
* Gets information about HDFS encryption
*/
- private HadoopShims.HdfsEncryptionShim hdfsEncryptionShim;
+ private Map<URI, HadoopShims.HdfsEncryptionShim> hdfsEncryptionShims = Maps.newHashMap();
/**
* Lineage state.
@@ -438,20 +439,31 @@ public class SessionState {
}
public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim() throws HiveException {
- if (hdfsEncryptionShim == null) {
+ try {
+ return getHdfsEncryptionShim(FileSystem.get(sessionConf));
+ }
+ catch(HiveException hiveException) {
+ throw hiveException;
+ }
+ catch(Exception exception) {
+ throw new HiveException(exception);
+ }
+ }
+
+ public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim(FileSystem fs) throws HiveException {
+ if (!hdfsEncryptionShims.containsKey(fs.getUri())) {
try {
- FileSystem fs = FileSystem.get(sessionConf);
if ("hdfs".equals(fs.getUri().getScheme())) {
- hdfsEncryptionShim = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, sessionConf);
+ hdfsEncryptionShims.put(fs.getUri(), ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, sessionConf));
} else {
- LOG.debug("Could not get hdfsEncryptionShim, it is only applicable to hdfs filesystem.");
+ LOG.info("Could not get hdfsEncryptionShim, it is only applicable to hdfs filesystem.");
}
} catch (Exception e) {
throw new HiveException(e);
}
}
- return hdfsEncryptionShim;
+ return hdfsEncryptionShims.get(fs.getUri());
}
// SessionState is not available in runtime and Hive.get().getConf() is not safe to call
http://git-wip-us.apache.org/repos/asf/hive/blob/4817d8bd/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index a4c7808..84b1174 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -1124,11 +1124,11 @@ public class Hadoop23Shims extends HadoopShimsSecure {
@Override
public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException {
- EncryptionZone zone1, zone2;
-
- zone1 = hdfsAdmin.getEncryptionZoneForPath(path1);
- zone2 = hdfsAdmin.getEncryptionZoneForPath(path2);
+ return equivalentEncryptionZones(hdfsAdmin.getEncryptionZoneForPath(path1),
+ hdfsAdmin.getEncryptionZoneForPath(path2));
+ }
+ private boolean equivalentEncryptionZones(EncryptionZone zone1, EncryptionZone zone2) {
if (zone1 == null && zone2 == null) {
return true;
} else if (zone1 == null || zone2 == null) {
@@ -1139,6 +1139,19 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
@Override
+ public boolean arePathsOnSameEncryptionZone(Path path1, Path path2,
+ HadoopShims.HdfsEncryptionShim encryptionShim2) throws IOException {
+ if (!(encryptionShim2 instanceof Hadoop23Shims.HdfsEncryptionShim)) {
+ LOG.warn("EncryptionShim for path2 (" + path2 + ") is of unexpected type: " + encryptionShim2.getClass()
+ + ". Assuming path2 is on the same EncryptionZone as path1(" + path1 + ").");
+ return true;
+ }
+
+ return equivalentEncryptionZones(hdfsAdmin.getEncryptionZoneForPath(path1),
+ ((HdfsEncryptionShim)encryptionShim2).hdfsAdmin.getEncryptionZoneForPath(path2));
+ }
+
+ @Override
public int comparePathKeyStrength(Path path1, Path path2) throws IOException {
EncryptionZone zone1, zone2;
http://git-wip-us.apache.org/repos/asf/hive/blob/4817d8bd/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index 9b0bc35..04f075b 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -512,6 +512,17 @@ public interface HadoopShims {
public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException;
/**
+ * Checks if two HDFS paths are on the same encrypted or unencrypted zone.
+ *
+ * @param path1 Path to HDFS file system
+ * @param path2 Path to HDFS file system
+ * @param encryptionShim2 The encryption-shim corresponding to path2.
+ * @return True if both paths are in the same zone; False otherwise.
+ * @throws IOException If an error occurred attempting to get encryption information
+ */
+ public boolean arePathsOnSameEncryptionZone(Path path1, Path path2, HdfsEncryptionShim encryptionShim2) throws IOException;
+
+ /**
* Compares two encrypted path strengths.
*
* @param path1 HDFS path to compare.
@@ -566,6 +577,12 @@ public interface HadoopShims {
}
@Override
+ public boolean arePathsOnSameEncryptionZone(Path path1, Path path2, HdfsEncryptionShim encryptionShim2) throws IOException {
+ // Not supported.
+ return true;
+ }
+
+ @Override
public int comparePathKeyStrength(Path path1, Path path2) throws IOException {
/* not supported */
return 0;