You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by am...@apache.org on 2023/08/21 23:38:41 UTC
[iceberg] branch wip-s3sign-body updated (b67a832973 -> 8872c3982c)
This is an automated email from the ASF dual-hosted git repository.
amoghj pushed a change to branch wip-s3sign-body
in repository https://gitbox.apache.org/repos/asf/iceberg.git
discard b67a832973 AWS: Update S3V4RestSignerClient to send body for DeleteObjects requests
new 8872c3982c AWS: Update S3V4RestSignerClient to send body for DeleteObjects requests
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (b67a832973)
\
N -- N -- N refs/heads/wip-s3sign-body (8872c3982c)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
[iceberg] 01/01: AWS: Update S3V4RestSignerClient to send body for DeleteObjects requests
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amoghj pushed a commit to branch wip-s3sign-body
in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit 8872c3982cd6a6a1be7cafcfaa9b8db38801b5d4
Author: amogh-jahagirdar <am...@tabular.io>
AuthorDate: Mon Aug 21 16:35:44 2023 -0700
AWS: Update S3V4RestSignerClient to send body for DeleteObjects requests
---
.../aws/s3/signer/S3V4RestSignerClient.java | 34 ++++++++++++++++++++--
.../iceberg/aws/s3/signer/TestS3RestSigner.java | 20 +++++++++++++
2 files changed, 51 insertions(+), 3 deletions(-)
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java
index a334221a25..703bf9a829 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java
@@ -21,6 +21,9 @@ package org.apache.iceberg.aws.s3.signer;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalListener;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
import java.net.URI;
import java.time.Duration;
import java.util.Collections;
@@ -55,6 +58,8 @@ import software.amazon.awssdk.auth.signer.params.AwsS3V4SignerParams;
import software.amazon.awssdk.core.checksums.SdkChecksum;
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import software.amazon.awssdk.http.SdkHttpFullRequest;
+import software.amazon.awssdk.http.SdkHttpMethod;
+import software.amazon.awssdk.utils.IoUtils;
@Value.Immutable
public abstract class S3V4RestSignerClient
@@ -279,15 +284,16 @@ public abstract class S3V4RestSignerClient
AwsS3V4SignerParams signerParams =
extractSignerParams(AwsS3V4SignerParams.builder(), executionAttributes).build();
- S3SignRequest remoteSigningRequest =
+ ImmutableS3SignRequest.Builder remoteSigningRequestBuilder =
ImmutableS3SignRequest.builder()
.method(request.method().name())
.region(signerParams.signingRegion().id())
.uri(request.getUri())
.headers(request.headers())
- .properties(requestPropertiesSupplier().get())
- .build();
+ .properties(requestPropertiesSupplier().get());
+ addBodyToRequestIfNeeded(request, remoteSigningRequestBuilder);
+ ImmutableS3SignRequest remoteSigningRequest = remoteSigningRequestBuilder.build();
Key cacheKey = Key.from(remoteSigningRequest);
SignedComponent cachedSignedComponent = SIGNED_COMPONENT_CACHE.getIfPresent(cacheKey);
SignedComponent signedComponent;
@@ -328,6 +334,28 @@ public abstract class S3V4RestSignerClient
return mutableRequest.build();
}
+ private void addBodyToRequestIfNeeded(
+ SdkHttpFullRequest request, ImmutableS3SignRequest.Builder requestBuilder) {
+ if (request.contentStreamProvider().isPresent()) {
+ if (shouldAddBodyToRequest(request)) {
+ try (InputStream is = request.contentStreamProvider().get().newStream()) {
+ requestBuilder.body(IoUtils.toUtf8String(is));
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to set body for S3 sign request", e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Only add body for DeleteObjectsRequest. Refer to
+ * https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html#API_DeleteObjects_RequestSyntax
+ */
+ private boolean shouldAddBodyToRequest(SdkHttpFullRequest request) {
+ return request.method() == SdkHttpMethod.POST
+ && request.rawQueryParameters().containsKey("delete");
+ }
+
private void reconstructHeaders(
Map<String, List<String>> signedAndUnsignedHeaders,
SdkHttpFullRequest.Builder mutableRequest) {
diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java b/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java
index 1e44e53318..b572b482dc 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java
@@ -20,6 +20,7 @@ package org.apache.iceberg.aws.s3.signer;
import static org.assertj.core.api.Assertions.assertThat;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
@@ -56,8 +57,11 @@ import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.Delete;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
@@ -177,6 +181,22 @@ public class TestS3RestSigner {
PutObjectRequest.builder().bucket(BUCKET).key("some/key").build(), Paths.get("/etc/hosts"));
}
+ @Test
+ public void validateDeleteObjects() {
+ Path sourcePath = Paths.get("/etc/hosts");
+ s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key("some/key1").build(), sourcePath);
+ s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key("some/key2").build(), sourcePath);
+
+ Delete objectsToDelete =
+ Delete.builder()
+ .objects(
+ ObjectIdentifier.builder().key("some/key1").build(),
+ ObjectIdentifier.builder().key("some/key2").build())
+ .build();
+
+ s3.deleteObjects(DeleteObjectsRequest.builder().bucket(BUCKET).delete(objectsToDelete).build());
+ }
+
@Test
public void validateListPrefix() {
s3.listObjectsV2(ListObjectsV2Request.builder().bucket(BUCKET).prefix("some/prefix/").build());