You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2019/08/23 06:15:49 UTC
[hadoop] branch trunk updated: HDDS-1948. S3 MPU can't be created
with octet-stream content-type (#1266)
This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new edd7085 HDDS-1948. S3 MPU can't be created with octet-stream content-type (#1266)
edd7085 is described below
commit edd708527d34d0bf3b09dc35a7f645f49e7becb3
Author: Elek, Márton <el...@users.noreply.github.com>
AuthorDate: Fri Aug 23 08:15:43 2019 +0200
HDDS-1948. S3 MPU can't be created with octet-stream content-type (#1266)
---
.../apache/hadoop/ozone/s3/HeaderPreprocessor.java | 31 ++++++++++--
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 5 ++
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 4 ++
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 56 ++++++++--------------
.../s3/endpoint/TestAbortMultipartUpload.java | 2 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 4 +-
.../hadoop/ozone/s3/endpoint/TestListParts.java | 2 +-
.../s3/endpoint/TestMultipartUploadComplete.java | 4 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 4 +-
9 files changed, 63 insertions(+), 49 deletions(-)
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java
index 0a1480a..db94bbb 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java
@@ -17,39 +17,60 @@
*/
package org.apache.hadoop.ozone.s3;
+import javax.annotation.Priority;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerRequestFilter;
import javax.ws.rs.container.PreMatching;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.ext.Provider;
import java.io.IOException;
/**
* Filter to adjust request headers for compatible reasons.
+ *
+ * It should be executed AFTER signature check (VirtualHostStyleFilter) as the
+ * original Content-Type could be part of the base of the signature.
*/
-
@Provider
@PreMatching
+@Priority(VirtualHostStyleFilter.PRIORITY
+ + S3GatewayHttpServer.FILTER_PRIORITY_DO_AFTER)
public class HeaderPreprocessor implements ContainerRequestFilter {
+ public static final String MULTIPART_UPLOAD_MARKER = "ozone/mpu";
+
@Override
public void filter(ContainerRequestContext requestContext) throws
IOException {
- if (requestContext.getUriInfo().getQueryParameters()
- .containsKey("delete")) {
+ MultivaluedMap<String, String> queryParameters =
+ requestContext.getUriInfo().getQueryParameters();
+
+ if (queryParameters.containsKey("delete")) {
//aws cli doesn't send proper Content-Type and by default POST requests
//processed as form-url-encoded. Here we can fix this.
requestContext.getHeaders()
.putSingle("Content-Type", MediaType.APPLICATION_XML);
}
- if (requestContext.getUriInfo().getQueryParameters()
- .containsKey("uploadId")) {
+ if (queryParameters.containsKey("uploadId")) {
//aws cli doesn't send proper Content-Type and by default POST requests
//processed as form-url-encoded. Here we can fix this.
requestContext.getHeaders()
.putSingle("Content-Type", MediaType.APPLICATION_XML);
+ } else if (queryParameters.containsKey("uploads")) {
+ // uploads defined but uploadId is not --> this is the creation of the
+ // multi-part-upload requests.
+ //
+ //In AWS SDK for go uses application/octet-stream which also
+ //should be fixed to route the request to the right jaxrs method.
+ //
+ //Should be empty instead of XML as the body is empty which can not be
+ //serialized as as CompleteMultipartUploadRequest
+ requestContext.getHeaders()
+ .putSingle("Content-Type", MULTIPART_UPLOAD_MARKER);
}
+
}
}
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java
index f20b928..f3d8341 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java
@@ -27,6 +27,11 @@ import org.apache.hadoop.hdds.server.BaseHttpServer;
*/
public class S3GatewayHttpServer extends BaseHttpServer {
+ /**
+ * Default offset between two filters.
+ */
+ public static final int FILTER_PRIORITY_DO_AFTER = 50;
+
public S3GatewayHttpServer(Configuration conf,
String name) throws IOException {
super(conf, name);
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java
index 50014fe..9ce98e1 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.s3;
+import javax.annotation.Priority;
import javax.inject.Inject;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerRequestFilter;
@@ -46,8 +47,11 @@ import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_DOMAIN_NA
@Provider
@PreMatching
+@Priority(VirtualHostStyleFilter.PRIORITY)
public class VirtualHostStyleFilter implements ContainerRequestFilter {
+ public static final int PRIORITY = 100;
+
private static final Logger LOG = LoggerFactory.getLogger(
VirtualHostStyleFilter.class);
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index b520e7b..70bfb7f 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.s3.endpoint;
+import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
@@ -58,6 +59,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
+import org.apache.hadoop.ozone.s3.HeaderPreprocessor;
import org.apache.hadoop.ozone.s3.SignedChunksInputStream;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
@@ -417,33 +419,19 @@ public class ObjectEndpoint extends EndpointBase {
}
+ /**
+ * Initialize MultiPartUpload request.
+ * <p>
+ * Note: the specific content type is set by the HeaderPreprocessor.
+ */
@POST
@Produces(MediaType.APPLICATION_XML)
- public Response multipartUpload(
+ @Consumes(HeaderPreprocessor.MULTIPART_UPLOAD_MARKER)
+ public Response initializeMultipartUpload(
@PathParam("bucket") String bucket,
- @PathParam("path") String key,
- @QueryParam("uploads") String uploads,
- @QueryParam("uploadId") @DefaultValue("") String uploadID,
- CompleteMultipartUploadRequest request) throws IOException, OS3Exception {
- if (!uploadID.equals("")) {
- //Complete Multipart upload request.
- return completeMultipartUpload(bucket, key, uploadID, request);
- } else {
- // Initiate Multipart upload request.
- return initiateMultipartUpload(bucket, key);
- }
- }
-
- /**
- * Initiate Multipart upload request.
- * @param bucket
- * @param key
- * @return Response
- * @throws IOException
- * @throws OS3Exception
- */
- private Response initiateMultipartUpload(String bucket, String key) throws
- IOException, OS3Exception {
+ @PathParam("path") String key
+ )
+ throws IOException, OS3Exception {
try {
OzoneBucket ozoneBucket = getBucket(bucket);
String storageType = headers.getHeaderString(STORAGE_CLASS_HEADER);
@@ -473,7 +461,6 @@ public class ObjectEndpoint extends EndpointBase {
multipartUploadInitiateResponse.setKey(key);
multipartUploadInitiateResponse.setUploadID(multipartInfo.getUploadID());
-
return Response.status(Status.OK).entity(
multipartUploadInitiateResponse).build();
} catch (IOException ex) {
@@ -484,18 +471,15 @@ public class ObjectEndpoint extends EndpointBase {
}
/**
- * Complete Multipart upload request.
- * @param bucket
- * @param key
- * @param uploadID
- * @param multipartUploadRequest
- * @return Response
- * @throws IOException
- * @throws OS3Exception
+ * Complete a multipart upload.
*/
- private Response completeMultipartUpload(String bucket, String key, String
- uploadID, CompleteMultipartUploadRequest multipartUploadRequest) throws
- IOException, OS3Exception {
+ @POST
+ @Produces(MediaType.APPLICATION_XML)
+ public Response completeMultipartUpload(@PathParam("bucket") String bucket,
+ @PathParam("path") String key,
+ @QueryParam("uploadId") @DefaultValue("") String uploadID,
+ CompleteMultipartUploadRequest multipartUploadRequest)
+ throws IOException, OS3Exception {
OzoneBucket ozoneBucket = getBucket(bucket);
Map<Integer, String> partsMap = new TreeMap<>();
List<CompleteMultipartUploadRequest.Part> partList =
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
index 76d4a12..912a769 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
@@ -56,7 +56,7 @@ public class TestAbortMultipartUpload {
rest.setHeaders(headers);
rest.setClient(client);
- Response response = rest.multipartUpload(bucket, key, "", "", null);
+ Response response = rest.initializeMultipartUpload(bucket, key);
assertEquals(response.getStatus(), 200);
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
index 6f48ecb..212721a 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
@@ -60,7 +60,7 @@ public class TestInitiateMultipartUpload {
rest.setHeaders(headers);
rest.setClient(client);
- Response response = rest.multipartUpload(bucket, key, "", "", null);
+ Response response = rest.initializeMultipartUpload(bucket, key);
assertEquals(response.getStatus(), 200);
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
@@ -69,7 +69,7 @@ public class TestInitiateMultipartUpload {
String uploadID = multipartUploadInitiateResponse.getUploadID();
// Calling again should return different uploadID.
- response = rest.multipartUpload(bucket, key, "", "", null);
+ response = rest.initializeMultipartUpload(bucket, key);
assertEquals(response.getStatus(), 200);
multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
index ac6aa72..21545ec 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
@@ -61,7 +61,7 @@ public class TestListParts {
REST.setHeaders(headers);
REST.setClient(client);
- Response response = REST.multipartUpload(BUCKET, KEY, "", "", null);
+ Response response = REST.initializeMultipartUpload(BUCKET, KEY);
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
index 8be6131..b9e3885 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
@@ -70,7 +70,7 @@ public class TestMultipartUploadComplete {
private String initiateMultipartUpload(String key) throws IOException,
OS3Exception {
- Response response = REST.multipartUpload(BUCKET, key, "", "", null);
+ Response response = REST.initializeMultipartUpload(BUCKET, key);
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
@@ -99,7 +99,7 @@ public class TestMultipartUploadComplete {
private void completeMultipartUpload(String key,
CompleteMultipartUploadRequest completeMultipartUploadRequest,
String uploadID) throws IOException, OS3Exception {
- Response response = REST.multipartUpload(BUCKET, key, "", uploadID,
+ Response response = REST.completeMultipartUpload(BUCKET, key, uploadID,
completeMultipartUploadRequest);
assertEquals(response.getStatus(), 200);
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
index 120fbb2..3e91a77 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
@@ -67,7 +67,7 @@ public class TestPartUpload {
@Test
public void testPartUpload() throws Exception {
- Response response = REST.multipartUpload(BUCKET, KEY, "", "", null);
+ Response response = REST.initializeMultipartUpload(BUCKET, KEY);
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
@@ -86,7 +86,7 @@ public class TestPartUpload {
@Test
public void testPartUploadWithOverride() throws Exception {
- Response response = REST.multipartUpload(BUCKET, KEY, "", "", null);
+ Response response = REST.initializeMultipartUpload(BUCKET, KEY);
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org