You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2023/05/22 09:38:52 UTC
[hadoop] 12/12: fixes issues after rebase
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch feature-HADOOP-18073-s3a-sdk-upgrade
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 75220b773873ed8919eace3bacfcf351b0110828
Author: Ahmar Suhail <ah...@amazon.co.uk>
AuthorDate: Wed May 17 13:35:28 2023 +0100
fixes issues after rebase
---
.../hadoop/fs/s3a/audit/impl/LoggingAuditor.java | 30 ++++++++++++----------
.../hadoop/fs/s3a/audit/AbstractAuditingTest.java | 27 ++++++++++++++++---
.../fs/s3a/audit/TestHttpReferrerAuditHeader.java | 12 ++++++---
3 files changed, 48 insertions(+), 21 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
index 602e0ef8ecb..3a2d9d7f823 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
@@ -29,6 +29,8 @@ import software.amazon.awssdk.core.SdkRequest;
import software.amazon.awssdk.core.interceptor.Context;
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import software.amazon.awssdk.http.SdkHttpRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -260,7 +262,8 @@ public class LoggingAuditor
/**
* Attach Range of data for GetObject Request.
- * @param request given get object request
+ * @param request the sdk request to be modified
+ * @param executionAttributes execution attributes for this request
*/
private void attachRangeFromRequest(SdkHttpRequest request,
ExecutionAttributes executionAttributes) {
@@ -370,12 +373,13 @@ public class LoggingAuditor
public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
ExecutionAttributes executionAttributes) {
SdkHttpRequest httpRequest = context.httpRequest();
+ SdkRequest sdkRequest = context.request();
- // attach range for GetObject requests
- attachRangeFromRequest(httpRequest, executionAttributes);
+ // attach range for GetObject requests
+ attachRangeFromRequest(httpRequest, executionAttributes);
- // for delete op, attach the number of files to delete
- attachDeleteKeySizeAttribute(request);
+ // for delete op, attach the number of files to delete
+ attachDeleteKeySizeAttribute(sdkRequest);
// build the referrer header
final String header = referrer.buildHttpReferrer();
@@ -397,9 +401,9 @@ public class LoggingAuditor
}
// now see if the request is actually a blocked multipart request
- if (!isMultipartUploadEnabled && isRequestMultipartIO(httpRequest)) {
+ if (!isMultipartUploadEnabled && isRequestMultipartIO(sdkRequest)) {
throw new AuditOperationRejectedException("Multipart IO request "
- + httpRequest + " rejected " + header);
+ + sdkRequest + " rejected " + header);
}
return httpRequest;
@@ -409,16 +413,16 @@ public class LoggingAuditor
* For delete requests, attach delete key size as a referrer attribute.
*
* @param request the request object.
- * @param <T> type of the request.
*/
- private <T extends AmazonWebServiceRequest> void attachDeleteKeySizeAttribute(T request) {
+ private void attachDeleteKeySizeAttribute(SdkRequest request) {
+
if (request instanceof DeleteObjectsRequest) {
- int keySize = ((DeleteObjectsRequest) request).getKeys().size();
- this.set(DELETE_KEYS_SIZE, String.valueOf(keySize));
+ int keySize = ((DeleteObjectsRequest) request).delete().objects().size();
+ referrer.set(DELETE_KEYS_SIZE, String.valueOf(keySize));
} else if (request instanceof DeleteObjectRequest) {
- String key = ((DeleteObjectRequest) request).getKey();
+ String key = ((DeleteObjectRequest) request).key();
if (key != null && key.length() > 0) {
- this.set(DELETE_KEYS_SIZE, "1");
+ referrer.set(DELETE_KEYS_SIZE, "1");
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java
index 0f6421d1bc4..5c33f19270e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java
@@ -34,8 +34,11 @@ import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import software.amazon.awssdk.core.interceptor.InterceptorContext;
import software.amazon.awssdk.http.SdkHttpMethod;
import software.amazon.awssdk.http.SdkHttpRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+
import org.junit.After;
import org.junit.Before;
import org.slf4j.Logger;
@@ -288,15 +291,31 @@ public abstract class AbstractAuditingTest extends AbstractHadoopTestBase {
* @param keys keys to be provided in the bulk delete request.
* @return a processed request.
*/
- protected DeleteObjectsRequest headForBulkDelete(String... keys) {
+ protected SdkHttpRequest headForBulkDelete(String... keys) {
if (keys == null || keys.length == 0) {
return null;
}
- List<DeleteObjectsRequest.KeyVersion> keysToDelete = Arrays
+
+ List<ObjectIdentifier> keysToDelete = Arrays
.stream(keys)
- .map(DeleteObjectsRequest.KeyVersion::new)
+ .map(key -> ObjectIdentifier.builder().key(key).build())
.collect(Collectors.toList());
- return manager.beforeExecution(requestFactory.newBulkDeleteRequest(keysToDelete));
+
+ ExecutionAttributes executionAttributes = ExecutionAttributes.builder().build();
+
+ SdkHttpRequest.Builder httpRequestBuilder =
+ SdkHttpRequest.builder().uri(URI.create("https://test")).method(SdkHttpMethod.POST);
+
+ DeleteObjectsRequest deleteObjectsRequest =
+ requestFactory.newBulkDeleteRequestBuilder(keysToDelete).build();
+
+ InterceptorContext context = InterceptorContext.builder()
+ .request(deleteObjectsRequest)
+ .httpRequest(httpRequestBuilder.build())
+ .build();
+
+ manager.beforeExecution(context, executionAttributes);
+ return manager.modifyHttpRequest(context, executionAttributes);
}
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java
index 430e1370c03..7f8dd043261 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java
@@ -350,16 +350,20 @@ public class TestHttpReferrerAuditHeader extends AbstractAuditingTest {
public void testHttpReferrerForBulkDelete() throws Throwable {
AuditSpan span = span();
long ts = span.getTimestamp();
- DeleteObjectsRequest request = headForBulkDelete(
+ SdkHttpRequest request = headForBulkDelete(
"key_01",
"key_02",
"key_03");
- Map<String, String> headers
- = request.getCustomRequestHeaders();
+ Map<String, List<String>> headers
+ = request.headers();
assertThat(headers)
.describedAs("Custom headers")
.containsKey(HEADER_REFERRER);
- String header = headers.get(HEADER_REFERRER);
+ List<String> headerValues = headers.get(HEADER_REFERRER);
+ assertThat(headerValues)
+ .describedAs("Multiple referrer headers")
+ .hasSize(1);
+ String header = headerValues.get(0);
LOG.info("Header is {}", header);
Map<String, String> params
= HttpReferrerAuditHeader.extractQueryParameters(header);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org