You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2024/10/07 15:21:14 UTC
Change in asterixdb[master]: WIP
From Hussain Towaileb <hu...@gmail.com>:
Hussain Towaileb has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18946 )
Change subject: WIP
......................................................................
WIP
Change-Id: I30933ac3fef0ae2fb09a88a02bd89fd5087b7071
---
M asterixdb/asterix-external-data/pom.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
4 files changed, 168 insertions(+), 10 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/46/18946/1
diff --git a/asterixdb/asterix-external-data/pom.xml b/asterixdb/asterix-external-data/pom.xml
index 6a3f891..2327005 100644
--- a/asterixdb/asterix-external-data/pom.xml
+++ b/asterixdb/asterix-external-data/pom.xml
@@ -449,6 +449,10 @@
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
+ <artifactId>sts</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
</dependency>
<dependency>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 104f301..d4bae11 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -73,7 +73,6 @@
import org.apache.asterix.external.library.msgpack.MessagePackUtils;
import org.apache.asterix.external.util.ExternalDataConstants.ParquetOptions;
import org.apache.asterix.external.util.aws.s3.S3Constants;
-import org.apache.asterix.external.util.aws.s3.S3Utils;
import org.apache.asterix.external.util.azure.blob_storage.AzureConstants;
import org.apache.asterix.external.util.google.gcs.GCSConstants;
import org.apache.asterix.om.types.ARecordType;
@@ -638,7 +637,7 @@
switch (type) {
case ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3:
- S3Utils.validateProperties(configuration, srcLoc, collector);
+ // S3Utils.validateProperties(configuration, srcLoc, collector);
break;
case ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_BLOB:
validateAzureBlobProperties(configuration, srcLoc, collector, appCtx);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
index a62b346..76b030d 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
@@ -28,6 +28,9 @@
public static final String ACCESS_KEY_ID_FIELD_NAME = "accessKeyId";
public static final String SECRET_ACCESS_KEY_FIELD_NAME = "secretAccessKey";
public static final String SESSION_TOKEN_FIELD_NAME = "sessionToken";
+ public static final String TRUST_USER_OR_ROLE_FIELD_NAME = "roleOrUser";
+ public static final String ROLE_ARN_FIELD_NAME = "roleArn";
+ public static final String EXTERNAL_ID_FIELD_NAME = "externalId";
public static final String SERVICE_END_POINT_FIELD_NAME = "serviceEndpoint";
// AWS S3 specific error codes
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
index 6a16913..2d10aea 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
@@ -28,6 +28,7 @@
import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_INTERNAL_ERROR;
import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_METHOD_NOT_IMPLEMENTED;
import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_SLOW_DOWN;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.EXTERNAL_ID_FIELD_NAME;
import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ACCESS_KEY_ID;
import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ANONYMOUS_ACCESS;
import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_CREDENTIAL_PROVIDER_KEY;
@@ -40,9 +41,11 @@
import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_TEMP_ACCESS;
import static org.apache.asterix.external.util.aws.s3.S3Constants.INSTANCE_PROFILE_FIELD_NAME;
import static org.apache.asterix.external.util.aws.s3.S3Constants.REGION_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.ROLE_ARN_FIELD_NAME;
import static org.apache.asterix.external.util.aws.s3.S3Constants.SECRET_ACCESS_KEY_FIELD_NAME;
import static org.apache.asterix.external.util.aws.s3.S3Constants.SERVICE_END_POINT_FIELD_NAME;
import static org.apache.asterix.external.util.aws.s3.S3Constants.SESSION_TOKEN_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.TRUST_USER_OR_ROLE_FIELD_NAME;
import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
import java.net.URI;
@@ -52,6 +55,7 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.UUID;
import java.util.function.BiPredicate;
import java.util.regex.Matcher;
@@ -91,6 +95,10 @@
import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.model.S3Response;
import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
+import software.amazon.awssdk.services.sts.StsClient;
+import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
+import software.amazon.awssdk.services.sts.model.AssumeRoleResponse;
+import software.amazon.awssdk.services.sts.model.Credentials;
public class S3Utils {
private S3Utils() {
@@ -111,6 +119,9 @@
public static S3Client buildAwsS3Client(Map<String, String> configuration) throws CompilationException {
// TODO(Hussain): Need to ensure that all required parameters are present in a previous step
String instanceProfile = configuration.get(INSTANCE_PROFILE_FIELD_NAME);
+ String trust = configuration.get(TRUST_USER_OR_ROLE_FIELD_NAME);
+ String arnRole = configuration.get(ROLE_ARN_FIELD_NAME);
+ String externalId = configuration.get(EXTERNAL_ID_FIELD_NAME);
String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
String sessionToken = configuration.get(SESSION_TOKEN_FIELD_NAME);
@@ -119,12 +130,6 @@
S3ClientBuilder builder = S3Client.builder();
- // Credentials
- AwsCredentialsProvider credentialsProvider =
- buildCredentialsProvider(instanceProfile, accessKeyId, secretAccessKey, sessionToken);
-
- builder.credentialsProvider(credentialsProvider);
-
// Validate the region
List<Region> regions = S3Client.serviceMetadata().regions();
Optional<Region> selectedRegion = regions.stream().filter(region -> region.id().equals(regionId)).findFirst();
@@ -134,6 +139,12 @@
}
builder.region(selectedRegion.get());
+ // Credentials
+ AwsCredentialsProvider credentialsProvider = buildCredentialsProvider2(configuration, selectedRegion.get(),
+ instanceProfile, trust, arnRole, externalId, accessKeyId, secretAccessKey, sessionToken);
+
+ builder.credentialsProvider(credentialsProvider);
+
// Validate the service endpoint if present
if (serviceEndpoint != null) {
try {
@@ -209,6 +220,56 @@
return credentialsProvider;
}
+ public static AwsCredentialsProvider buildCredentialsProvider2(Map<String, String> configuration, Region region,
+ String instanceProfile, String trust, String arnRole, String externalId, String accessKeyId,
+ String secretAccessKey, String sessionToken) throws CompilationException {
+
+ AwsCredentialsProvider credentialsProvider;
+ if (noAuth(configuration)) {
+ credentialsProvider = AnonymousCredentialsProvider.create();
+ } else if (arnRole != null) {
+ // TODO: Do auth validation and use existing credentials if exist already, if not, assume the role
+ return assumeRoleAndGetCredentials(trust, arnRole, accessKeyId, secretAccessKey, externalId, region);
+ } else if (instanceProfile != null) {
+ // only "true" value is allowed
+ if (!instanceProfile.equalsIgnoreCase("true")) {
+ throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE, INSTANCE_PROFILE_FIELD_NAME, "true");
+ }
+
+ String fieldName = getNonNull(configuration, ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME,
+ SESSION_TOKEN_FIELD_NAME, ROLE_ARN_FIELD_NAME, EXTERNAL_ID_FIELD_NAME);
+ if (fieldName != null) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, fieldName,
+ INSTANCE_PROFILE_FIELD_NAME);
+ }
+ credentialsProvider = InstanceProfileCredentialsProvider.create();
+ } else if (accessKeyId != null || secretAccessKey != null) {
+ // accessKeyId authentication
+ if (accessKeyId == null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
+ SECRET_ACCESS_KEY_FIELD_NAME);
+ }
+ if (secretAccessKey == null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, SECRET_ACCESS_KEY_FIELD_NAME,
+ ACCESS_KEY_ID_FIELD_NAME);
+ }
+
+ // use session token if provided
+ if (sessionToken != null) {
+ credentialsProvider = StaticCredentialsProvider
+ .create(AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken));
+ } else {
+ credentialsProvider =
+ StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
+ }
+ } else {
+ // if only session token is provided, accessKeyId is required
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
+ SESSION_TOKEN_FIELD_NAME);
+ }
+ return credentialsProvider;
+ }
+
/**
* Builds the S3 client using the provided configuration
*
@@ -278,10 +339,21 @@
throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
}
- // Both parameters should be passed, or neither should be passed (for anonymous/no auth)
+ String arnRole = configuration.get(ROLE_ARN_FIELD_NAME);
+ String externalId = configuration.get(EXTERNAL_ID_FIELD_NAME);
String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
- if (accessKeyId == null || secretAccessKey == null) {
+
+ if (arnRole != null) {
+ String fieldName = getNonNull(configuration, ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME);
+ if (fieldName != null) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, fieldName,
+ INSTANCE_PROFILE_FIELD_NAME);
+ }
+ } else if (externalId != null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ROLE_ARN_FIELD_NAME,
+ EXTERNAL_ID_FIELD_NAME);
+ } else if (accessKeyId == null || secretAccessKey == null) {
// If one is passed, the other is required
if (accessKeyId != null) {
throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, SECRET_ACCESS_KEY_FIELD_NAME,
@@ -576,4 +648,75 @@
allObjects.put("folders", folders);
return allObjects;
}
+
+ private static AwsCredentialsProvider assumeRoleAndGetCredentials(String trust, String arnRole, String accessKeyId,
+ String secretAccessKey, String externalId, Region region) throws CompilationException {
+ try {
+ String sessionToken = null;
+ // if ("role".equals(trust)) {
+ // AssumeRoleRequest.Builder builder = AssumeRoleRequest.builder();
+ // builder.roleArn("arn:aws:iam::506345531828:role/i-can-be-trusted-role");
+ // builder.roleSessionName(UUID.randomUUID().toString());
+ // if (externalId != null) {
+ // builder.externalId(externalId);
+ // }
+ // AssumeRoleRequest request = builder.build();
+ // StaticCredentialsProvider credentialsProvider =
+ // StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
+ // try (StsClient stsClient =
+ // StsClient.builder().region(region).build()) {
+ // AssumeRoleResponse response = stsClient.assumeRole(request);
+ // Credentials credentials = response.credentials();
+ // accessKeyId = credentials.accessKeyId();
+ // secretAccessKey = credentials.secretAccessKey();
+ // sessionToken = credentials.sessionToken();
+ // }
+ // }
+
+ AssumeRoleRequest.Builder builder = AssumeRoleRequest.builder();
+ builder.roleArn(arnRole);
+ builder.roleSessionName(UUID.randomUUID().toString());
+ if (externalId != null) {
+ builder.externalId(externalId);
+ }
+ AssumeRoleRequest request = builder.build();
+
+ AwsCredentialsProvider credentialsProvider;
+ // if ("role".equals(trust)) {
+ credentialsProvider = InstanceProfileCredentialsProvider.create();
+ // } else {
+ // if (sessionToken == null) {
+ // credentialsProvider =
+ // StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
+ // } else {
+ // credentialsProvider = StaticCredentialsProvider
+ // .create(AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken));
+ // }
+ // }
+
+ try (StsClient stsClient =
+ StsClient.builder().region(region).credentialsProvider(credentialsProvider).build()) {
+ AssumeRoleResponse response = stsClient.assumeRole(request);
+ Credentials credentials = response.credentials();
+ return StaticCredentialsProvider.create(AwsSessionCredentials.create(credentials.accessKeyId(),
+ credentials.secretAccessKey(), credentials.sessionToken()));
+ }
+ } catch (SdkException ex) {
+ throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex, getMessageOrToString(ex));
+ }
+ }
+
+ private static boolean noAuth(Map<String, String> configuration) {
+ return getNonNull(configuration, INSTANCE_PROFILE_FIELD_NAME, ROLE_ARN_FIELD_NAME, EXTERNAL_ID_FIELD_NAME,
+ ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME, SESSION_TOKEN_FIELD_NAME) == null;
+ }
+
+ private static String getNonNull(Map<String, String> configuration, String... fieldNames) {
+ for (String fieldName : fieldNames) {
+ if (configuration.get(fieldName) != null) {
+ return fieldName;
+ }
+ }
+ return null;
+ }
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18946
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I30933ac3fef0ae2fb09a88a02bd89fd5087b7071
Gerrit-Change-Number: 18946
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <hu...@gmail.com>
Gerrit-MessageType: newchange