You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2020/01/07 11:17:47 UTC
[hadoop] branch trunk updated: HADOOP-16645. S3A Delegation Token
extension point to use StoreContext.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2bbf73f HADOOP-16645. S3A Delegation Token extension point to use StoreContext.
2bbf73f is described below
commit 2bbf73f1df96ebe7e6fcf64a724ae846230a8487
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Tue Jan 7 11:15:53 2020 +0000
HADOOP-16645. S3A Delegation Token extension point to use StoreContext.
Contributed by Steve Loughran.
This is part of the ongoing refactoring of the S3A codebase, with the
delegation token support (HADOOP-14556) no longer given a direct reference
to the owning S3AFileSystem. Instead it gets a StoreContext and a new
interface, DelegationOperations, to access those operations offered by S3AFS
which are specifically needed by the DT bindings.
The sole operation needed is listAWSPolicyRules(), which is used to allow
S3A FS and the S3Guard metastore to return the AWS policy rules needed to
access their specific services/buckets/tables, allowing the AssumedRole
delegation token to be locked down.
As further restructuring takes place, that interface's implementation
can be moved to wherever the new home for those operations ends up.
Although it changes the API of an extension point, that feature (S3
Delegation Tokens) has not shipped; backwards compatibility is not a
problem except for anyone who has implemented DT support against trunk.
To those developers: sorry.
Change-Id: I770f58b49ff7634a34875ba37b7d51c94d7c21da
---
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 25 ++++++++++-
.../fs/s3a/auth/delegation/AbstractDTService.java | 48 +++++++++++++---------
.../delegation/AbstractDelegationTokenBinding.java | 4 +-
.../s3a/auth/delegation/DelegationOperations.java | 28 +++++++++++++
.../delegation/FullCredentialsTokenBinding.java | 7 ++--
.../fs/s3a/auth/delegation/RoleTokenBinding.java | 3 +-
.../s3a/auth/delegation/S3ADelegationTokens.java | 16 +++++---
.../s3a/auth/delegation/SessionTokenBinding.java | 2 +-
.../s3a/auth/delegation/AbstractDelegationIT.java | 5 ++-
.../auth/delegation/ITestRoleDelegationTokens.java | 4 +-
.../delegation/ITestSessionDelegationTokens.java | 5 ++-
11 files changed, 110 insertions(+), 37 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index cc1371a..b85b9c1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -96,6 +96,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Globber;
import org.apache.hadoop.fs.s3a.auth.SignerManager;
+import org.apache.hadoop.fs.s3a.auth.delegation.DelegationOperations;
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy;
import org.apache.hadoop.fs.s3a.impl.ContextAccessors;
@@ -541,7 +542,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
LOG.debug("Using delegation tokens");
S3ADelegationTokens tokens = new S3ADelegationTokens();
this.delegationTokens = Optional.of(tokens);
- tokens.bindToFileSystem(getCanonicalUri(), this);
+ tokens.bindToFileSystem(getCanonicalUri(),
+ createStoreContext(),
+ createDelegationOperations());
tokens.init(conf);
tokens.start();
// switch to the DT provider and bypass all other configured
@@ -575,6 +578,26 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
/**
+ * Implementation of all operations used by delegation tokens.
+ */
+ private class DelegationOperationsImpl implements DelegationOperations {
+
+ @Override
+ public List<RoleModel.Statement> listAWSPolicyRules(final Set<AccessLevel> access) {
+ return S3AFileSystem.this.listAWSPolicyRules(access);
+ }
+ }
+
+ /**
+ * Create an instance of the delegation operations.
+ * @return callbacks for DT support.
+ */
+ @VisibleForTesting
+ public DelegationOperations createDelegationOperations() {
+ return new DelegationOperationsImpl();
+ }
+
+ /**
* Set the encryption secrets for requests.
* @param secrets secrets
*/
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDTService.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDTService.java
index dcb83c2..b121154 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDTService.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDTService.java
@@ -24,7 +24,7 @@ import java.net.URI;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.impl.StoreContext;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
@@ -53,21 +53,26 @@ public abstract class AbstractDTService
/**
* URI of the filesystem.
- * Valid after {@link #bindToFileSystem(URI, S3AFileSystem)}.
+ * Valid after {@link #bindToFileSystem(URI, StoreContext, DelegationOperations)}.
*/
private URI canonicalUri;
/**
- * The owning filesystem.
- * Valid after {@link #bindToFileSystem(URI, S3AFileSystem)}.
+ * Owner of the filesystem.
+ * Valid after {@link #bindToFileSystem(URI, StoreContext, DelegationOperations)}.
*/
- private S3AFileSystem fileSystem;
+ private UserGroupInformation owner;
/**
- * Owner of the filesystem.
- * Valid after {@link #bindToFileSystem(URI, S3AFileSystem)}.
+ * Store Context for callbacks into the FS.
+ * Valid after {@link #bindToFileSystem(URI, StoreContext, DelegationOperations)}.
*/
- private UserGroupInformation owner;
+ private StoreContext storeContext;
+
+ /**
+ * Callbacks for DT-related operations.
+ */
+ private DelegationOperations policyProvider;
/**
* Protected constructor.
@@ -88,18 +93,21 @@ public abstract class AbstractDTService
* is not live for actual use and will not yet have interacted with
* AWS services.
* @param uri the canonical URI of the FS.
- * @param fs owning FS.
+ * @param context store context
+ * @param delegationOperations delegation operations
* @throws IOException failure.
*/
public void bindToFileSystem(
final URI uri,
- final S3AFileSystem fs) throws IOException {
+ final StoreContext context,
+ final DelegationOperations delegationOperations) throws IOException {
requireServiceState(STATE.NOTINITED);
Preconditions.checkState(canonicalUri == null,
"bindToFileSystem called twice");
this.canonicalUri = requireNonNull(uri);
- this.fileSystem = requireNonNull(fs);
- this.owner = fs.getOwner();
+ this.storeContext = requireNonNull(context);
+ this.owner = context.getOwner();
+ this.policyProvider = delegationOperations;
}
/**
@@ -112,14 +120,6 @@ public abstract class AbstractDTService
}
/**
- * Get the owner of the FS.
- * @return the owner fs
- */
- protected S3AFileSystem getFileSystem() {
- return fileSystem;
- }
-
- /**
* Get the owner of this Service.
* @return owner; non-null after binding to an FS.
*/
@@ -127,6 +127,14 @@ public abstract class AbstractDTService
return owner;
}
+ protected StoreContext getStoreContext() {
+ return storeContext;
+ }
+
+ protected DelegationOperations getPolicyProvider() {
+ return policyProvider;
+ }
+
/**
* Require that the service is in a given state.
* @param state desired state.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java
index f940c40..fc7b095 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java
@@ -28,7 +28,6 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.auth.RoleModel;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.token.SecretManager;
@@ -55,7 +54,8 @@ import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DURAT
* instance which created it --which itself follows the lifecycle of the FS.
*
* One big difference is that
- * {@link #bindToFileSystem(URI, S3AFileSystem)} will be called
+ * {@link AbstractDTService#bindToFileSystem(URI, org.apache.hadoop.fs.s3a.impl.StoreContext, DelegationOperations)}
+ * will be called
* before the {@link #init(Configuration)} operation, this is where
* the owning FS is passed in.
*
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/DelegationOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/DelegationOperations.java
new file mode 100644
index 0000000..d46276a
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/DelegationOperations.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.auth.delegation;
+
+/**
+ * All operations used for delegation which aren't in the store context.
+ * Initially this is just the AWS policy operations; it's
+ * here so that any future evolution isn't going to break the signatures of
+ * external DT implementations.
+ */
+public interface DelegationOperations extends AWSPolicyProvider {
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java
index 24020e3..d80e780 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java
@@ -120,7 +120,7 @@ public class FullCredentialsTokenBinding extends
"Full Credentials Token Binding",
new MarshalledCredentialProvider(
FULL_TOKEN,
- getFileSystem().getUri(),
+ getStoreContext().getFsURI(),
getConfig(),
awsCredentials,
MarshalledCredentials.CredentialTypeRequired.AnyNonEmpty));
@@ -159,9 +159,10 @@ public class FullCredentialsTokenBinding extends
convertTokenIdentifier(retrievedIdentifier,
FullCredentialsTokenIdentifier.class);
return new AWSCredentialProviderList(
- "", new MarshalledCredentialProvider(
+ "Full Credentials Token Binding",
+ new MarshalledCredentialProvider(
FULL_TOKEN,
- getFileSystem().getUri(),
+ getStoreContext().getFsURI(),
getConfig(),
tokenIdentifier.getMarshalledCredentials(),
MarshalledCredentials.CredentialTypeRequired.AnyNonEmpty));
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java
index db72b63..3c86033 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java
@@ -110,7 +110,8 @@ public class RoleTokenBinding extends SessionTokenBinding {
return new AWSCredentialProviderList(
"Role Token Binding",
new MarshalledCredentialProvider(
- COMPONENT, getFileSystem().getUri(),
+ COMPONENT,
+ getStoreContext().getFsURI(),
getConfig(),
marshalledCredentials,
MarshalledCredentials.CredentialTypeRequired.SessionOnly));
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java
index a7720c2..5005436 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java
@@ -34,9 +34,9 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3AInstrumentation;
import org.apache.hadoop.fs.s3a.auth.RoleModel;
+import org.apache.hadoop.fs.s3a.impl.StoreContext;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
@@ -154,11 +154,13 @@ public class S3ADelegationTokens extends AbstractDTService {
}
@Override
- public void bindToFileSystem(final URI uri, final S3AFileSystem fs)
+ public void bindToFileSystem(final URI uri,
+ final StoreContext context,
+ final DelegationOperations delegationOperations)
throws IOException {
- super.bindToFileSystem(uri, fs);
+ super.bindToFileSystem(uri, context, delegationOperations);
service = getTokenService(getCanonicalUri());
- stats = fs.getInstrumentation().newDelegationTokenStatistics();
+ stats = context.getInstrumentation().newDelegationTokenStatistics();
}
/**
@@ -179,7 +181,9 @@ public class S3ADelegationTokens extends AbstractDTService {
SessionTokenBinding.class,
AbstractDelegationTokenBinding.class);
tokenBinding = binding.newInstance();
- tokenBinding.bindToFileSystem(getCanonicalUri(), getFileSystem());
+ tokenBinding.bindToFileSystem(getCanonicalUri(),
+ getStoreContext(),
+ getPolicyProvider());
tokenBinding.init(conf);
tokenBindingName = tokenBinding.getKind().toString();
LOG.debug("Filesystem {} is using delegation tokens of kind {}",
@@ -415,7 +419,7 @@ public class S3ADelegationTokens extends AbstractDTService {
"Null encryption secrets");
// this isn't done in in advance as it needs S3Guard initialized in the
// filesystem before it can generate complete policies.
- List<RoleModel.Statement> statements = getFileSystem()
+ List<RoleModel.Statement> statements = getPolicyProvider()
.listAWSPolicyRules(ACCESS_POLICY);
Optional<RoleModel.Policy> rolePolicy =
statements.isEmpty() ?
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java
index 6ee32a8..3e49031 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java
@@ -226,7 +226,7 @@ public class SessionTokenBinding extends AbstractDelegationTokenBinding {
"Session Token Binding",
new MarshalledCredentialProvider(
SESSION_TOKEN,
- getFileSystem().getUri(),
+ getStoreContext().getFsURI(),
getConfig(),
marshalledCredentials,
MarshalledCredentials.CredentialTypeRequired.SessionOnly));
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationIT.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationIT.java
index d9c18d9..67ed3d5 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationIT.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationIT.java
@@ -205,7 +205,10 @@ public abstract class AbstractDelegationIT extends AbstractS3ATestBase {
throws IOException {
S3AFileSystem fs = getFileSystem();
S3ADelegationTokens tokens = new S3ADelegationTokens();
- tokens.bindToFileSystem(fs.getCanonicalUri(), fs);
+ tokens.bindToFileSystem(
+ fs.getCanonicalUri(),
+ fs.createStoreContext(),
+ fs.createDelegationOperations());
tokens.init(conf);
return tokens;
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestRoleDelegationTokens.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestRoleDelegationTokens.java
index 48182f7..1085c26 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestRoleDelegationTokens.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestRoleDelegationTokens.java
@@ -93,7 +93,9 @@ public class ITestRoleDelegationTokens extends ITestSessionDelegationTokens {
conf.unset(DelegationConstants.DELEGATION_TOKEN_ROLE_ARN);
try (S3ADelegationTokens delegationTokens2 = new S3ADelegationTokens()) {
final S3AFileSystem fs = getFileSystem();
- delegationTokens2.bindToFileSystem(fs.getUri(), fs);
+ delegationTokens2.bindToFileSystem(fs.getUri(),
+ fs.createStoreContext(),
+ fs.createDelegationOperations());
delegationTokens2.init(conf);
delegationTokens2.start();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
index 79011b5..7a70ac9 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
@@ -263,7 +263,10 @@ public class ITestSessionDelegationTokens extends AbstractDelegationIT {
TemporaryAWSCredentialsProvider.NAME);
session.setSecretsInConfiguration(conf);
try(S3ADelegationTokens delegationTokens2 = new S3ADelegationTokens()) {
- delegationTokens2.bindToFileSystem(fs.getCanonicalUri(), fs);
+ delegationTokens2.bindToFileSystem(
+ fs.getCanonicalUri(),
+ fs.createStoreContext(),
+ fs.createDelegationOperations());
delegationTokens2.init(conf);
delegationTokens2.start();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org