You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zy...@apache.org on 2018/07/10 04:38:54 UTC
[32/50] [abbrv] hbase git commit: HBASE-20244 NoSuchMethodException
when retrieving private method decryptEncryptedDataEncryptionKey from
DFSClient
HBASE-20244 NoSuchMethodException when retrieving private method decryptEncryptedDataEncryptionKey from DFSClient
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0f237841
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0f237841
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0f237841
Branch: refs/heads/HBASE-18477
Commit: 0f23784182ab88649de340d75804e0ff20dcd0fc
Parents: 2d2d995
Author: Ted Yu <te...@apache.org>
Authored: Tue Jul 3 18:22:10 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jul 3 22:15:18 2018 +0800
----------------------------------------------------------------------
.../FanOutOneBlockAsyncDFSOutputSaslHelper.java | 108 +++++++++++++------
1 file changed, 73 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0f237841/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
index af2ccf9..ef6c1ca 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
@@ -17,34 +17,11 @@
*/
package org.apache.hadoop.hbase.io.asyncfs;
-import static org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleState.READER_IDLE;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY;
+import static org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleState.READER_IDLE;
-import org.apache.hbase.thirdparty.com.google.common.base.Charsets;
-import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
-import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
+import com.google.protobuf.ByteString;
import com.google.protobuf.CodedOutputStream;
-
-import org.apache.hbase.thirdparty.io.netty.buffer.ByteBuf;
-import org.apache.hbase.thirdparty.io.netty.buffer.ByteBufOutputStream;
-import org.apache.hbase.thirdparty.io.netty.buffer.CompositeByteBuf;
-import org.apache.hbase.thirdparty.io.netty.buffer.Unpooled;
-import org.apache.hbase.thirdparty.io.netty.channel.Channel;
-import org.apache.hbase.thirdparty.io.netty.channel.ChannelDuplexHandler;
-import org.apache.hbase.thirdparty.io.netty.channel.ChannelHandlerContext;
-import org.apache.hbase.thirdparty.io.netty.channel.ChannelOutboundHandlerAdapter;
-import org.apache.hbase.thirdparty.io.netty.channel.ChannelPipeline;
-import org.apache.hbase.thirdparty.io.netty.channel.ChannelPromise;
-import org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler;
-import org.apache.hbase.thirdparty.io.netty.handler.codec.LengthFieldBasedFrameDecoder;
-import org.apache.hbase.thirdparty.io.netty.handler.codec.MessageToByteEncoder;
-import org.apache.hbase.thirdparty.io.netty.handler.codec.protobuf.ProtobufDecoder;
-import org.apache.hbase.thirdparty.io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder;
-import org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateEvent;
-import org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler;
-import org.apache.hbase.thirdparty.io.netty.util.concurrent.Promise;
-
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
@@ -60,7 +37,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-
import javax.security.auth.callback.Callback;
import javax.security.auth.callback.CallbackHandler;
import javax.security.auth.callback.NameCallback;
@@ -71,7 +47,6 @@ import javax.security.sasl.RealmChoiceCallback;
import javax.security.sasl.Sasl;
import javax.security.sasl.SaslClient;
import javax.security.sasl.SaslException;
-
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -80,13 +55,9 @@ import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoCodec;
import org.apache.hadoop.crypto.Decryptor;
import org.apache.hadoop.crypto.Encryptor;
+import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import org.apache.hadoop.fs.FileEncryptionInfo;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.protobuf.ByteString;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -102,6 +73,32 @@ import org.apache.hadoop.security.SaslPropertiesResolver;
import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.common.base.Charsets;
+import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
+import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
+import org.apache.hbase.thirdparty.io.netty.buffer.ByteBuf;
+import org.apache.hbase.thirdparty.io.netty.buffer.ByteBufOutputStream;
+import org.apache.hbase.thirdparty.io.netty.buffer.CompositeByteBuf;
+import org.apache.hbase.thirdparty.io.netty.buffer.Unpooled;
+import org.apache.hbase.thirdparty.io.netty.channel.Channel;
+import org.apache.hbase.thirdparty.io.netty.channel.ChannelDuplexHandler;
+import org.apache.hbase.thirdparty.io.netty.channel.ChannelHandlerContext;
+import org.apache.hbase.thirdparty.io.netty.channel.ChannelOutboundHandlerAdapter;
+import org.apache.hbase.thirdparty.io.netty.channel.ChannelPipeline;
+import org.apache.hbase.thirdparty.io.netty.channel.ChannelPromise;
+import org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler;
+import org.apache.hbase.thirdparty.io.netty.handler.codec.LengthFieldBasedFrameDecoder;
+import org.apache.hbase.thirdparty.io.netty.handler.codec.MessageToByteEncoder;
+import org.apache.hbase.thirdparty.io.netty.handler.codec.protobuf.ProtobufDecoder;
+import org.apache.hbase.thirdparty.io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder;
+import org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateEvent;
+import org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler;
+import org.apache.hbase.thirdparty.io.netty.util.concurrent.Promise;
/**
* Helper class for adding sasl support for {@link FanOutOneBlockAsyncDFSOutput}.
@@ -226,10 +223,10 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
};
}
- private static TransparentCryptoHelper createTransparentCryptoHelper()
+ private static TransparentCryptoHelper createTransparentCryptoHelper27()
throws NoSuchMethodException {
Method decryptEncryptedDataEncryptionKeyMethod = DFSClient.class
- .getDeclaredMethod("decryptEncryptedDataEncryptionKey", FileEncryptionInfo.class);
+ .getDeclaredMethod("decryptEncryptedDataEncryptionKey", FileEncryptionInfo.class);
decryptEncryptedDataEncryptionKeyMethod.setAccessible(true);
return new TransparentCryptoHelper() {
@@ -238,7 +235,37 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
DFSClient client) throws IOException {
try {
KeyVersion decryptedKey =
- (KeyVersion) decryptEncryptedDataEncryptionKeyMethod.invoke(client, feInfo);
+ (KeyVersion) decryptEncryptedDataEncryptionKeyMethod.invoke(client, feInfo);
+ CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf, feInfo.getCipherSuite());
+ Encryptor encryptor = cryptoCodec.createEncryptor();
+ encryptor.init(decryptedKey.getMaterial(), feInfo.getIV());
+ return encryptor;
+ } catch (InvocationTargetException e) {
+ Throwables.propagateIfPossible(e.getTargetException(), IOException.class);
+ throw new RuntimeException(e.getTargetException());
+ } catch (GeneralSecurityException e) {
+ throw new IOException(e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+
+ private static TransparentCryptoHelper createTransparentCryptoHelper28()
+ throws ClassNotFoundException, NoSuchMethodException {
+ Class<?> hdfsKMSUtilCls = Class.forName("org.apache.hadoop.hdfs.HdfsKMSUtil");
+ Method decryptEncryptedDataEncryptionKeyMethod = hdfsKMSUtilCls.getDeclaredMethod(
+ "decryptEncryptedDataEncryptionKey", FileEncryptionInfo.class, KeyProvider.class);
+ decryptEncryptedDataEncryptionKeyMethod.setAccessible(true);
+ return new TransparentCryptoHelper() {
+
+ @Override
+ public Encryptor createEncryptor(Configuration conf, FileEncryptionInfo feInfo,
+ DFSClient client) throws IOException {
+ try {
+ KeyVersion decryptedKey = (KeyVersion) decryptEncryptedDataEncryptionKeyMethod
+ .invoke(null, feInfo, client.getKeyProvider());
CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf, feInfo.getCipherSuite());
Encryptor encryptor = cryptoCodec.createEncryptor();
encryptor.init(decryptedKey.getMaterial(), feInfo.getIV());
@@ -255,6 +282,17 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
};
}
+ private static TransparentCryptoHelper createTransparentCryptoHelper()
+ throws NoSuchMethodException, ClassNotFoundException {
+ try {
+ return createTransparentCryptoHelper27();
+ } catch (NoSuchMethodException e) {
+ LOG.debug("No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop 2.8+",
+ e);
+ }
+ return createTransparentCryptoHelper28();
+ }
+
static {
try {
SASL_ADAPTOR = createSaslAdaptor();