You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by GitBox <gi...@apache.org> on 2022/09/15 14:23:41 UTC
[GitHub] [iceberg] singhpk234 commented on a diff in pull request #5684: AWS: Refactor util methods for applying AWS clients configurations
singhpk234 commented on code in PR #5684:
URL: https://github.com/apache/iceberg/pull/5684#discussion_r971949892
##########
aws/src/main/java/org/apache/iceberg/aws/lakeformation/LakeFormationAwsClientFactory.java:
##########
@@ -64,7 +63,8 @@ public LakeFormationAwsClientFactory() {}
public void initialize(Map<String, String> catalogProperties) {
super.initialize(catalogProperties);
Preconditions.checkArgument(
- tags().stream().anyMatch(t -> t.key().equals(LF_AUTHORIZED_CALLER)),
+ awsProperties().stsClientAssumeRoleTags().stream()
+ .anyMatch(t -> t.key().equals(LF_AUTHORIZED_CALLER)),
Review Comment:
[minor]
```suggestion
.anyMatch(t -> LF_AUTHORIZED_CALLER.equals(t.key())),
```
##########
aws/src/main/java/org/apache/iceberg/aws/AwsProperties.java:
##########
@@ -726,13 +829,150 @@ public void setS3DeleteEnabled(boolean s3DeleteEnabled) {
this.isS3DeleteEnabled = s3DeleteEnabled;
}
- private Set<Tag> toTags(Map<String, String> properties, String prefix) {
+ private Set<Tag> toS3Tags(Map<String, String> properties, String prefix) {
Review Comment:
[minor] should we move the private functions below public funcs
##########
aws/src/main/java/org/apache/iceberg/aws/AwsProperties.java:
##########
@@ -467,39 +513,61 @@ public AwsProperties() {
this.isS3DeleteEnabled = S3_DELETE_ENABLED_DEFAULT;
this.s3BucketToAccessPointMapping = ImmutableMap.of();
this.s3PreloadClientEnabled = S3_PRELOAD_CLIENT_ENABLED_DEFAULT;
+ this.s3DualStackEnabled = S3_DUALSTACK_ENABLED_DEFAULT;
+ this.s3PathStyleAccess = S3FILEIO_PATH_STYLE_ACCESS_DEFAULT;
+ this.s3UseArnRegionEnabled = S3_USE_ARN_REGION_ENABLED_DEFAULT;
+ this.s3AccelerationEnabled = S3_ACCELERATION_ENABLED_DEFAULT;
this.glueCatalogId = null;
+ this.glueEndpoint = null;
this.glueCatalogSkipArchive = GLUE_CATALOG_SKIP_ARCHIVE_DEFAULT;
this.glueCatalogSkipNameValidation = GLUE_CATALOG_SKIP_NAME_VALIDATION_DEFAULT;
this.glueLakeFormationEnabled = GLUE_LAKEFORMATION_ENABLED_DEFAULT;
+ this.dynamoDbEndpoint = null;
this.dynamoDbTableName = DYNAMODB_TABLE_NAME_DEFAULT;
+
+ ValidationException.check(
+ s3KeyIdAccessKeyBothConfigured(),
+ "S3 client access key ID and secret access key must be set at the same time");
}
public AwsProperties(Map<String, String> properties) {
- this.s3FileIoSseType =
- properties.getOrDefault(
- AwsProperties.S3FILEIO_SSE_TYPE, AwsProperties.S3FILEIO_SSE_TYPE_NONE);
- this.s3FileIoSseKey = properties.get(AwsProperties.S3FILEIO_SSE_KEY);
- this.s3FileIoSseMd5 = properties.get(AwsProperties.S3FILEIO_SSE_MD5);
- if (AwsProperties.S3FILEIO_SSE_TYPE_CUSTOM.equals(s3FileIoSseType)) {
+ this.httpClientType =
+ PropertyUtil.propertyAsString(properties, HTTP_CLIENT_TYPE, HTTP_CLIENT_TYPE_DEFAULT);
+ this.stsClientAssumeRoleTags = toStsTags(properties, CLIENT_ASSUME_ROLE_TAGS_PREFIX);
+
+ this.clientAssumeRoleArn = properties.get(CLIENT_ASSUME_ROLE_ARN);
+ this.clientAssumeRoleTimeoutSec =
+ PropertyUtil.propertyAsInt(
+ properties, CLIENT_ASSUME_ROLE_TIMEOUT_SEC, CLIENT_ASSUME_ROLE_TIMEOUT_SEC_DEFAULT);
+ this.clientAssumeRoleExternalId = properties.get(CLIENT_ASSUME_ROLE_EXTERNAL_ID);
+ this.clientAssumeRoleRegion = properties.get(CLIENT_ASSUME_ROLE_REGION);
+
+ this.s3FileIoSseType = properties.getOrDefault(S3FILEIO_SSE_TYPE, S3FILEIO_SSE_TYPE_NONE);
+ this.s3FileIoSseKey = properties.get(S3FILEIO_SSE_KEY);
+ this.s3FileIoSseMd5 = properties.get(S3FILEIO_SSE_MD5);
+ this.s3AccessKeyId = properties.get(S3FILEIO_ACCESS_KEY_ID);
+ this.s3SecretAccessKey = properties.get(S3FILEIO_SECRET_ACCESS_KEY);
+ this.s3SessionToken = properties.get(S3FILEIO_SESSION_TOKEN);
+ if (S3FILEIO_SSE_TYPE_CUSTOM.equals(s3FileIoSseType)) {
Preconditions.checkNotNull(
s3FileIoSseKey, "Cannot initialize SSE-C S3FileIO with null encryption key");
Preconditions.checkNotNull(
s3FileIoSseMd5, "Cannot initialize SSE-C S3FileIO with null encryption key MD5");
}
+ this.s3Endpoint = properties.get(S3FILEIO_ENDPOINT);
+ this.glueEndpoint = properties.get(GLUE_CATALOG_ENDPOINT);
this.glueCatalogId = properties.get(GLUE_CATALOG_ID);
this.glueCatalogSkipArchive =
PropertyUtil.propertyAsBoolean(
- properties,
- AwsProperties.GLUE_CATALOG_SKIP_ARCHIVE,
- AwsProperties.GLUE_CATALOG_SKIP_ARCHIVE_DEFAULT);
+ properties, GLUE_CATALOG_SKIP_ARCHIVE, GLUE_CATALOG_SKIP_ARCHIVE_DEFAULT);
this.glueCatalogSkipNameValidation =
PropertyUtil.propertyAsBoolean(
properties,
- AwsProperties.GLUE_CATALOG_SKIP_NAME_VALIDATION,
- AwsProperties.GLUE_CATALOG_SKIP_NAME_VALIDATION_DEFAULT);
+ GLUE_CATALOG_SKIP_NAME_VALIDATION,
+ GLUE_CATALOG_SKIP_NAME_VALIDATION_DEFAULT);
Review Comment:
[nit] let's move them to one line like L#565
##########
aws/src/test/java/org/apache/iceberg/aws/TestAwsClientFactories.java:
##########
@@ -56,24 +52,6 @@ public void testLoadCustom() {
"should load custom class", AwsClientFactories.from(properties) instanceof CustomFactory);
}
- @Test
Review Comment:
[question] is the coverage for the code-path, covered in some other UT ? If not would recommend adapting this UT to the changes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org