You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/10/16 23:38:05 UTC
[21/50] [abbrv] hadoop git commit: HADOOP-14445. Addendum: Use
DelegationTokenIssuer to create KMS delegation tokens that can authenticate
to all KMS instances.
HADOOP-14445. Addendum: Use DelegationTokenIssuer to create KMS delegation tokens that can authenticate to all KMS instances.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6fc72a0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6fc72a0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6fc72a0
Branch: refs/heads/HDFS-12943
Commit: b6fc72a0250ac3f2341ebe8a14d19b073e6224c8
Parents: ee1c80e
Author: Xiao Chen <xi...@apache.org>
Authored: Mon Oct 15 10:50:27 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Mon Oct 15 10:51:55 2018 -0700
----------------------------------------------------------------------
.../KeyProviderDelegationTokenExtension.java | 2 +-
.../crypto/key/KeyProviderTokenIssuer.java | 2 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 2 +-
.../security/token/DelegationTokenIssuer.java | 3 +-
.../apache/hadoop/fs/TestFilterFileSystem.java | 2 +-
.../org/apache/hadoop/fs/TestHarFileSystem.java | 2 +-
.../hadoop/hdfs/DistributedFileSystem.java | 2 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 11 ++++++-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 32 +++++++++++---------
9 files changed, 34 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc72a0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
index 29c5bcd..05d99ed 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.org.apache.hadoop.security.token.DelegationTokenIssuer;
+import org.apache.hadoop.security.token.DelegationTokenIssuer;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc72a0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderTokenIssuer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderTokenIssuer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderTokenIssuer.java
index 81caff4..187bee6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderTokenIssuer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderTokenIssuer.java
@@ -22,7 +22,7 @@ import java.net.URI;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.security.token.org.apache.hadoop.security.token.DelegationTokenIssuer;
+import org.apache.hadoop.security.token.DelegationTokenIssuer;
/**
* File systems that support Encryption Zones have to implement this interface.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc72a0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 3d40b6a..fe4159b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -64,7 +64,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.org.apache.hadoop.security.token.DelegationTokenIssuer;
+import org.apache.hadoop.security.token.DelegationTokenIssuer;
import org.apache.hadoop.util.ClassUtil;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc72a0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/org/apache/hadoop/security/token/DelegationTokenIssuer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/org/apache/hadoop/security/token/DelegationTokenIssuer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/org/apache/hadoop/security/token/DelegationTokenIssuer.java
index 90e72b9..70a53b7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/org/apache/hadoop/security/token/DelegationTokenIssuer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/org/apache/hadoop/security/token/DelegationTokenIssuer.java
@@ -15,13 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.security.token.org.apache.hadoop.security.token;
+package org.apache.hadoop.security.token;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.token.Token;
import java.io.IOException;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc72a0/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
index a766cfb4..9e01aef 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.org.apache.hadoop.security.token.DelegationTokenIssuer;
+import org.apache.hadoop.security.token.DelegationTokenIssuer;
import org.apache.hadoop.util.Progressable;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc72a0/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
index 870a828..025b831 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.org.apache.hadoop.security.token.DelegationTokenIssuer;
+import org.apache.hadoop.security.token.DelegationTokenIssuer;
import org.apache.hadoop.util.Progressable;
import org.junit.Assert;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc72a0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 12bc73c..ca1546c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -103,7 +103,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.org.apache.hadoop.security.token.DelegationTokenIssuer;
+import org.apache.hadoop.security.token.DelegationTokenIssuer;
import org.apache.hadoop.util.ChunkedArrayList;
import org.apache.hadoop.util.Progressable;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc72a0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index b7325ba..c139cb0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -118,7 +118,7 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.TokenSelector;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
-import org.apache.hadoop.security.token.org.apache.hadoop.security.token.DelegationTokenIssuer;
+import org.apache.hadoop.security.token.DelegationTokenIssuer;
import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.KMSUtil;
import org.apache.hadoop.util.Progressable;
@@ -173,6 +173,7 @@ public class WebHdfsFileSystem extends FileSystem
private Set<String> restCsrfMethodsToIgnore;
private DFSOpsCountStatistics storageStatistics;
+ private KeyProvider testProvider;
/**
* Return the protocol scheme for the FileSystem.
@@ -1949,6 +1950,9 @@ public class WebHdfsFileSystem extends FileSystem
@Override
public KeyProvider getKeyProvider() throws IOException {
+ if (testProvider != null) {
+ return testProvider;
+ }
URI keyProviderUri = getKeyProviderUri();
if (keyProviderUri == null) {
return null;
@@ -1956,6 +1960,11 @@ public class WebHdfsFileSystem extends FileSystem
return KMSUtil.createKeyProviderFromUri(getConf(), keyProviderUri);
}
+ @VisibleForTesting
+ public void setTestProvider(KeyProvider kp) {
+ testProvider = kp;
+ }
+
/**
* This class is used for opening, reading, and seeking files while using the
* WebHdfsFileSystem. This class will invoke the retry policy when performing
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc72a0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index d8524ae..465e925 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -97,6 +97,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.DelegationTokenIssuer;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.DelegationTokenExtension;
@@ -118,7 +119,6 @@ import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Matchers.anyShort;
import static org.mockito.Mockito.withSettings;
-import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyString;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
@@ -1339,11 +1339,13 @@ public class TestEncryptionZones {
byte[] testIdentifier = "Test identifier for delegation token".getBytes();
@SuppressWarnings("rawtypes")
- Token<?> testToken = new Token(testIdentifier, new byte[0],
+ Token testToken = new Token(testIdentifier, new byte[0],
new Text(), new Text());
- Mockito.when(((DelegationTokenExtension)keyProvider).
- addDelegationTokens(anyString(), (Credentials)any())).
- thenReturn(new Token<?>[] { testToken });
+ Mockito.when(((DelegationTokenIssuer)keyProvider).
+ getCanonicalServiceName()).thenReturn("service");
+ Mockito.when(((DelegationTokenIssuer)keyProvider).
+ getDelegationToken(anyString())).
+ thenReturn(testToken);
dfs.getClient().setKeyProvider(keyProvider);
@@ -1353,7 +1355,7 @@ public class TestEncryptionZones {
Arrays.asList(tokens));
Assert.assertEquals(2, tokens.length);
Assert.assertEquals(tokens[1], testToken);
- Assert.assertEquals(1, creds.numberOfTokens());
+ Assert.assertEquals(2, creds.numberOfTokens());
}
/**
@@ -2106,22 +2108,22 @@ public class TestEncryptionZones {
Mockito.when(keyProvider.getConf()).thenReturn(conf);
byte[] testIdentifier = "Test identifier for delegation token".getBytes();
- Token<?> testToken = new Token(testIdentifier, new byte[0],
+ Token testToken = new Token(testIdentifier, new byte[0],
new Text("kms-dt"), new Text());
- Mockito.when(((DelegationTokenExtension) keyProvider)
- .addDelegationTokens(anyString(), (Credentials) any()))
- .thenReturn(new Token<?>[] {testToken});
-
- WebHdfsFileSystem webfsSpy = Mockito.spy(webfs);
- Mockito.doReturn(keyProvider).when(webfsSpy).getKeyProvider();
+ Mockito.when(((DelegationTokenIssuer)keyProvider).
+ getCanonicalServiceName()).thenReturn("service");
+ Mockito.when(((DelegationTokenIssuer)keyProvider).
+ getDelegationToken(anyString())).
+ thenReturn(testToken);
+ webfs.setTestProvider(keyProvider);
Credentials creds = new Credentials();
final Token<?>[] tokens =
- webfsSpy.addDelegationTokens("JobTracker", creds);
+ webfs.addDelegationTokens("JobTracker", creds);
Assert.assertEquals(2, tokens.length);
Assert.assertEquals(tokens[1], testToken);
- Assert.assertEquals(1, creds.numberOfTokens());
+ Assert.assertEquals(2, creds.numberOfTokens());
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org