You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2024/10/07 15:21:14 UTC

Change in asterixdb[master]: WIP

From Hussain Towaileb <hu...@gmail.com>:

Hussain Towaileb has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18946 )


Change subject: WIP
......................................................................

WIP

Change-Id: I30933ac3fef0ae2fb09a88a02bd89fd5087b7071
---
M asterixdb/asterix-external-data/pom.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
4 files changed, 168 insertions(+), 10 deletions(-)



  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/46/18946/1

diff --git a/asterixdb/asterix-external-data/pom.xml b/asterixdb/asterix-external-data/pom.xml
index 6a3f891..2327005 100644
--- a/asterixdb/asterix-external-data/pom.xml
+++ b/asterixdb/asterix-external-data/pom.xml
@@ -449,6 +449,10 @@
     </dependency>
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
+      <artifactId>sts</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
       <artifactId>s3</artifactId>
     </dependency>
     <dependency>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 104f301..d4bae11 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -73,7 +73,6 @@
 import org.apache.asterix.external.library.msgpack.MessagePackUtils;
 import org.apache.asterix.external.util.ExternalDataConstants.ParquetOptions;
 import org.apache.asterix.external.util.aws.s3.S3Constants;
-import org.apache.asterix.external.util.aws.s3.S3Utils;
 import org.apache.asterix.external.util.azure.blob_storage.AzureConstants;
 import org.apache.asterix.external.util.google.gcs.GCSConstants;
 import org.apache.asterix.om.types.ARecordType;
@@ -638,7 +637,7 @@
 
         switch (type) {
             case ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3:
-                S3Utils.validateProperties(configuration, srcLoc, collector);
+                //                S3Utils.validateProperties(configuration, srcLoc, collector);
                 break;
             case ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_BLOB:
                 validateAzureBlobProperties(configuration, srcLoc, collector, appCtx);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
index a62b346..76b030d 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
@@ -28,6 +28,9 @@
     public static final String ACCESS_KEY_ID_FIELD_NAME = "accessKeyId";
     public static final String SECRET_ACCESS_KEY_FIELD_NAME = "secretAccessKey";
     public static final String SESSION_TOKEN_FIELD_NAME = "sessionToken";
+    public static final String TRUST_USER_OR_ROLE_FIELD_NAME = "roleOrUser";
+    public static final String ROLE_ARN_FIELD_NAME = "roleArn";
+    public static final String EXTERNAL_ID_FIELD_NAME = "externalId";
     public static final String SERVICE_END_POINT_FIELD_NAME = "serviceEndpoint";
 
     // AWS S3 specific error codes
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
index 6a16913..2d10aea 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
@@ -28,6 +28,7 @@
 import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_INTERNAL_ERROR;
 import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_METHOD_NOT_IMPLEMENTED;
 import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_SLOW_DOWN;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.EXTERNAL_ID_FIELD_NAME;
 import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ACCESS_KEY_ID;
 import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ANONYMOUS_ACCESS;
 import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_CREDENTIAL_PROVIDER_KEY;
@@ -40,9 +41,11 @@
 import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_TEMP_ACCESS;
 import static org.apache.asterix.external.util.aws.s3.S3Constants.INSTANCE_PROFILE_FIELD_NAME;
 import static org.apache.asterix.external.util.aws.s3.S3Constants.REGION_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.ROLE_ARN_FIELD_NAME;
 import static org.apache.asterix.external.util.aws.s3.S3Constants.SECRET_ACCESS_KEY_FIELD_NAME;
 import static org.apache.asterix.external.util.aws.s3.S3Constants.SERVICE_END_POINT_FIELD_NAME;
 import static org.apache.asterix.external.util.aws.s3.S3Constants.SESSION_TOKEN_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.TRUST_USER_OR_ROLE_FIELD_NAME;
 import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
 
 import java.net.URI;
@@ -52,6 +55,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.UUID;
 import java.util.function.BiPredicate;
 import java.util.regex.Matcher;
 
@@ -91,6 +95,10 @@
 import software.amazon.awssdk.services.s3.model.S3Object;
 import software.amazon.awssdk.services.s3.model.S3Response;
 import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
+import software.amazon.awssdk.services.sts.StsClient;
+import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
+import software.amazon.awssdk.services.sts.model.AssumeRoleResponse;
+import software.amazon.awssdk.services.sts.model.Credentials;
 
 public class S3Utils {
     private S3Utils() {
@@ -111,6 +119,9 @@
     public static S3Client buildAwsS3Client(Map<String, String> configuration) throws CompilationException {
         // TODO(Hussain): Need to ensure that all required parameters are present in a previous step
         String instanceProfile = configuration.get(INSTANCE_PROFILE_FIELD_NAME);
+        String trust = configuration.get(TRUST_USER_OR_ROLE_FIELD_NAME);
+        String arnRole = configuration.get(ROLE_ARN_FIELD_NAME);
+        String externalId = configuration.get(EXTERNAL_ID_FIELD_NAME);
         String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
         String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
         String sessionToken = configuration.get(SESSION_TOKEN_FIELD_NAME);
@@ -119,12 +130,6 @@
 
         S3ClientBuilder builder = S3Client.builder();
 
-        // Credentials
-        AwsCredentialsProvider credentialsProvider =
-                buildCredentialsProvider(instanceProfile, accessKeyId, secretAccessKey, sessionToken);
-
-        builder.credentialsProvider(credentialsProvider);
-
         // Validate the region
         List<Region> regions = S3Client.serviceMetadata().regions();
         Optional<Region> selectedRegion = regions.stream().filter(region -> region.id().equals(regionId)).findFirst();
@@ -134,6 +139,12 @@
         }
         builder.region(selectedRegion.get());
 
+        // Credentials
+        AwsCredentialsProvider credentialsProvider = buildCredentialsProvider2(configuration, selectedRegion.get(),
+                instanceProfile, trust, arnRole, externalId, accessKeyId, secretAccessKey, sessionToken);
+
+        builder.credentialsProvider(credentialsProvider);
+
         // Validate the service endpoint if present
         if (serviceEndpoint != null) {
             try {
@@ -209,6 +220,56 @@
         return credentialsProvider;
     }
 
+    public static AwsCredentialsProvider buildCredentialsProvider2(Map<String, String> configuration, Region region,
+            String instanceProfile, String trust, String arnRole, String externalId, String accessKeyId,
+            String secretAccessKey, String sessionToken) throws CompilationException {
+
+        AwsCredentialsProvider credentialsProvider;
+        if (noAuth(configuration)) {
+            credentialsProvider = AnonymousCredentialsProvider.create();
+        } else if (arnRole != null) {
+            // TODO: Do auth validation and use existing credentials if exist already, if not, assume the role
+            return assumeRoleAndGetCredentials(trust, arnRole, accessKeyId, secretAccessKey, externalId, region);
+        } else if (instanceProfile != null) {
+            // only "true" value is allowed
+            if (!instanceProfile.equalsIgnoreCase("true")) {
+                throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE, INSTANCE_PROFILE_FIELD_NAME, "true");
+            }
+
+            String fieldName = getNonNull(configuration, ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME,
+                    SESSION_TOKEN_FIELD_NAME, ROLE_ARN_FIELD_NAME, EXTERNAL_ID_FIELD_NAME);
+            if (fieldName != null) {
+                throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, fieldName,
+                        INSTANCE_PROFILE_FIELD_NAME);
+            }
+            credentialsProvider = InstanceProfileCredentialsProvider.create();
+        } else if (accessKeyId != null || secretAccessKey != null) {
+            // accessKeyId authentication
+            if (accessKeyId == null) {
+                throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
+                        SECRET_ACCESS_KEY_FIELD_NAME);
+            }
+            if (secretAccessKey == null) {
+                throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, SECRET_ACCESS_KEY_FIELD_NAME,
+                        ACCESS_KEY_ID_FIELD_NAME);
+            }
+
+            // use session token if provided
+            if (sessionToken != null) {
+                credentialsProvider = StaticCredentialsProvider
+                        .create(AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken));
+            } else {
+                credentialsProvider =
+                        StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
+            }
+        } else {
+            // if only session token is provided, accessKeyId is required
+            throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
+                    SESSION_TOKEN_FIELD_NAME);
+        }
+        return credentialsProvider;
+    }
+
     /**
      * Builds the S3 client using the provided configuration
      *
@@ -278,10 +339,21 @@
             throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
         }
 
-        // Both parameters should be passed, or neither should be passed (for anonymous/no auth)
+        String arnRole = configuration.get(ROLE_ARN_FIELD_NAME);
+        String externalId = configuration.get(EXTERNAL_ID_FIELD_NAME);
         String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
         String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
-        if (accessKeyId == null || secretAccessKey == null) {
+
+        if (arnRole != null) {
+            String fieldName = getNonNull(configuration, ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME);
+            if (fieldName != null) {
+                throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, fieldName,
+                        INSTANCE_PROFILE_FIELD_NAME);
+            }
+        } else if (externalId != null) {
+            throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ROLE_ARN_FIELD_NAME,
+                    EXTERNAL_ID_FIELD_NAME);
+        } else if (accessKeyId == null || secretAccessKey == null) {
             // If one is passed, the other is required
             if (accessKeyId != null) {
                 throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, SECRET_ACCESS_KEY_FIELD_NAME,
@@ -576,4 +648,75 @@
         allObjects.put("folders", folders);
         return allObjects;
     }
+
+    private static AwsCredentialsProvider assumeRoleAndGetCredentials(String trust, String arnRole, String accessKeyId,
+            String secretAccessKey, String externalId, Region region) throws CompilationException {
+        try {
+            String sessionToken = null;
+            //            if ("role".equals(trust)) {
+            //                AssumeRoleRequest.Builder builder = AssumeRoleRequest.builder();
+            //                builder.roleArn("arn:aws:iam::506345531828:role/i-can-be-trusted-role");
+            //                builder.roleSessionName(UUID.randomUUID().toString());
+            //                if (externalId != null) {
+            //                    builder.externalId(externalId);
+            //                }
+            //                AssumeRoleRequest request = builder.build();
+            //                StaticCredentialsProvider credentialsProvider =
+            //                        StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
+            //                try (StsClient stsClient =
+            //                        StsClient.builder().region(region).build()) {
+            //                    AssumeRoleResponse response = stsClient.assumeRole(request);
+            //                    Credentials credentials = response.credentials();
+            //                    accessKeyId = credentials.accessKeyId();
+            //                    secretAccessKey = credentials.secretAccessKey();
+            //                    sessionToken = credentials.sessionToken();
+            //                }
+            //            }
+
+            AssumeRoleRequest.Builder builder = AssumeRoleRequest.builder();
+            builder.roleArn(arnRole);
+            builder.roleSessionName(UUID.randomUUID().toString());
+            if (externalId != null) {
+                builder.externalId(externalId);
+            }
+            AssumeRoleRequest request = builder.build();
+
+            AwsCredentialsProvider credentialsProvider;
+            //            if ("role".equals(trust)) {
+            credentialsProvider = InstanceProfileCredentialsProvider.create();
+            //            } else {
+            //                if (sessionToken == null) {
+            //                    credentialsProvider =
+            //                            StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
+            //                } else {
+            //                    credentialsProvider = StaticCredentialsProvider
+            //                            .create(AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken));
+            //                }
+            //            }
+
+            try (StsClient stsClient =
+                    StsClient.builder().region(region).credentialsProvider(credentialsProvider).build()) {
+                AssumeRoleResponse response = stsClient.assumeRole(request);
+                Credentials credentials = response.credentials();
+                return StaticCredentialsProvider.create(AwsSessionCredentials.create(credentials.accessKeyId(),
+                        credentials.secretAccessKey(), credentials.sessionToken()));
+            }
+        } catch (SdkException ex) {
+            throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex, getMessageOrToString(ex));
+        }
+    }
+
+    private static boolean noAuth(Map<String, String> configuration) {
+        return getNonNull(configuration, INSTANCE_PROFILE_FIELD_NAME, ROLE_ARN_FIELD_NAME, EXTERNAL_ID_FIELD_NAME,
+                ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME, SESSION_TOKEN_FIELD_NAME) == null;
+    }
+
+    private static String getNonNull(Map<String, String> configuration, String... fieldNames) {
+        for (String fieldName : fieldNames) {
+            if (configuration.get(fieldName) != null) {
+                return fieldName;
+            }
+        }
+        return null;
+    }
 }

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18946
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I30933ac3fef0ae2fb09a88a02bd89fd5087b7071
Gerrit-Change-Number: 18946
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <hu...@gmail.com>
Gerrit-MessageType: newchange