You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2019/03/08 18:16:14 UTC
[hadoop] branch ozone-0.4 updated: HDDS-1213. Support plain text S3
MPU initialization request.
This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch ozone-0.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/ozone-0.4 by this push:
new 118986c HDDS-1213. Support plain text S3 MPU initialization request.
118986c is described below
commit 118986cde79905d20942bc68413eb00132b3a2f4
Author: Elek, Márton <el...@users.noreply.github.com>
AuthorDate: Fri Mar 8 18:55:31 2019 +0100
HDDS-1213. Support plain text S3 MPU initialization request.
(cherry picked from commit c072458e5d1ec88c8e4d76cd09623083b8ccfc03)
---
.../dist/src/main/smoketest/commonlib.robot | 8 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 135 ++++++++++-----------
.../dist/src/main/smoketest/s3/objectputget.robot | 4 +-
.../endpoint/PlainTextMultipartUploadReader.java | 66 ++++++++++
4 files changed, 139 insertions(+), 74 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot
index eb3a8bb..7f8cd4b 100644
--- a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot
@@ -28,4 +28,10 @@ Execute and checkrc
${rc} ${output} = Run And Return Rc And Output ${command}
Log ${output}
Should Be Equal As Integers ${rc} ${expected_error_code}
- [return] ${output}
\ No newline at end of file
+ [return] ${output}
+
+Compare files
+ [arguments] ${file1} ${file2}
+ ${checksumbefore} = Execute md5sum ${file1} | awk '{print $1}'
+ ${checksumafter} = Execute md5sum ${file2} | awk '{print $1}'
+ Should Be Equal ${checksumbefore} ${checksumafter}
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
index 1f69c9e..0133d50 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
@@ -22,11 +22,9 @@ Resource commonawslib.robot
Test Setup Setup s3 tests
*** Keywords ***
-Create Random file for mac
- Execute dd if=/dev/urandom of=/tmp/part1 bs=1m count=5
-
-Create Random file for linux
- Execute dd if=/dev/urandom of=/tmp/part1 bs=1M count=5
+Create Random file
+ [arguments] ${size_in_megabytes}
+ Execute dd if=/dev/urandom of=/tmp/part1 bs=1048576 count=${size_in_megabytes}
*** Variables ***
@@ -54,16 +52,13 @@ Test Multipart Upload
# upload we get error entity too small. So, considering further complete
# multipart upload, uploading each part as 5MB file, exception is for last part
- ${system} = Evaluate platform.system() platform
- Run Keyword if '${system}' == 'Darwin' Create Random file for mac
- Run Keyword if '${system}' == 'Linux' Create Random file for linux
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
- Should contain ${result} ETag
+ Run Keyword Create Random file 5
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
+ Should contain ${result} ETag
# override part
- Run Keyword if '${system}' == 'Darwin' Create Random file for mac
- Run Keyword if '${system}' == 'Linux' Create Random file for linux
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
- Should contain ${result} ETag
+ Run Keyword Create Random file 5
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
+ Should contain ${result} ETag
Test Multipart Upload Complete
@@ -74,17 +69,15 @@ Test Multipart Upload Complete
Should contain ${result} UploadId
#upload parts
- ${system} = Evaluate platform.system() platform
- Run Keyword if '${system}' == 'Darwin' Create Random file for mac
- Run Keyword if '${system}' == 'Linux' Create Random file for linux
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
- ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
- Should contain ${result} ETag
+ Run Keyword Create Random file 5
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
+ ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
+ Should contain ${result} ETag
Execute echo "Part2" > /tmp/part2
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
- ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
- Should contain ${result} ETag
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
+ ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
+ Should contain ${result} ETag
#complete multipart upload
${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey1 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
@@ -94,11 +87,8 @@ Test Multipart Upload Complete
#read file and check the key
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key multipartKey1 /tmp/multipartKey1.result
- Execute cat /tmp/part1 /tmp/part2 >> /tmp/multipartkey1
- ${checksumbefore} = Execute md5sum /tmp/multipartkey1 | awk '{print $1}'
- ${checksumafter} = Execute md5sum /tmp/multipartKey1.result | awk '{print $1}'
- Should Be Equal ${checksumbefore} ${checksumafter}
-
+ Execute cat /tmp/part1 /tmp/part2 >> /tmp/multipartKey1
+ Compare files /tmp/multipartKey1 /tmp/multipartKey1.result
Test Multipart Upload Complete Entity too small
${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key multipartKey2
@@ -109,14 +99,14 @@ Test Multipart Upload Complete Entity too small
#upload parts
Execute echo "Part1" > /tmp/part1
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
- ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
- Should contain ${result} ETag
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
+ ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
+ Should contain ${result} ETag
Execute echo "Part2" > /tmp/part2
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
- ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
- Should contain ${result} ETag
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
+ ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
+ Should contain ${result} ETag
#complete multipart upload
${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey2 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' 255
@@ -132,14 +122,14 @@ Test Multipart Upload Complete Invalid part
#upload parts
Execute echo "Part1" > /tmp/part1
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
- ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
- Should contain ${result} ETag
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
+ ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
+ Should contain ${result} ETag
Execute echo "Part2" > /tmp/part2
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
- ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
- Should contain ${result} ETag
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
+ ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
+ Should contain ${result} ETag
#complete multipart upload
${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255
@@ -158,9 +148,9 @@ Test abort Multipart upload with invalid uploadId
${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id "random" 255
Upload part with Incorrect uploadID
- Execute echo "Multipart upload" > /tmp/testfile
- ${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255
- Should contain ${result} NoSuchUpload
+ Execute echo "Multipart upload" > /tmp/testfile
+ ${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255
+ Should contain ${result} NoSuchUpload
Test list parts
#initiate multipart upload
@@ -171,37 +161,42 @@ Test list parts
Should contain ${result} UploadId
#upload parts
- ${system} = Evaluate platform.system() platform
- Run Keyword if '${system}' == 'Darwin' Create Random file for mac
- Run Keyword if '${system}' == 'Linux' Create Random file for linux
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
- ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
- Should contain ${result} ETag
-
- Execute echo "Part2" > /tmp/part2
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
- ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
- Should contain ${result} ETag
+ Run Keyword Create Random file 5
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
+ ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
+ Should contain ${result} ETag
+
+ Execute echo "Part2" > /tmp/part2
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
+ ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
+ Should contain ${result} ETag
#list parts
- ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID}
- ${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
- ${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[1].ETag' 0
- Should Be equal ${part1} ${eTag1}
- Should contain ${part2} ${eTag2}
- Should contain ${result} STANDARD
+ ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID}
+ ${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
+ ${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[1].ETag' 0
+ Should Be equal ${part1} ${eTag1}
+ Should contain ${part2} ${eTag2}
+ Should contain ${result} STANDARD
#list parts with max-items and next token
- ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1
- ${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
- ${token} = Execute and checkrc echo '${result}' | jq -r '.NextToken' 0
- Should Be equal ${part1} ${eTag1}
- Should contain ${result} STANDARD
+ ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1
+ ${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
+ ${token} = Execute and checkrc echo '${result}' | jq -r '.NextToken' 0
+ Should Be equal ${part1} ${eTag1}
+ Should contain ${result} STANDARD
- ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token ${token}
- ${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
- Should Be equal ${part2} ${eTag2}
- Should contain ${result} STANDARD
+ ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token ${token}
+ ${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
+ Should Be equal ${part2} ${eTag2}
+ Should contain ${result} STANDARD
#finally abort it
- ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} 0
+ ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} 0
+
+Test Multipart Upload with the simplified aws s3 cp API
+ Create Random file 22
+ Execute AWSS3Cli cp /tmp/part1 s3://${BUCKET}/mpyawscli
+ Execute AWSS3Cli cp s3://${BUCKET}/mpyawscli /tmp/part1.result
+ Execute AWSS3Cli rm s3://${BUCKET}/mpyawscli
+ Compare files /tmp/part1 /tmp/part1.result
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
index e42ffaf..1b2a504 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
@@ -42,9 +42,7 @@ Put object to s3
#This test depends on the previous test case. Can't be executes alone
Get object from s3
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 /tmp/testfile.result
- ${checksumbefore} = Execute md5sum /tmp/testfile | awk '{print $1}'
- ${checksumafter} = Execute md5sum /tmp/testfile.result | awk '{print $1}'
- Should Be Equal ${checksumbefore} ${checksumafter}
+ Compare files /tmp/testfile /tmp/testfile.result
Get Partial object from s3 with both start and endoffset
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-4 /tmp/testfile1.result
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PlainTextMultipartUploadReader.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PlainTextMultipartUploadReader.java
new file mode 100644
index 0000000..599b473
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PlainTextMultipartUploadReader.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.ext.MessageBodyReader;
+import javax.ws.rs.ext.Provider;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Type;
+
+/**
+ * Body reader to accept plain text MPU.
+ * <p>
+ * Aws s3 api sends a multipartupload request with the content type
+ * 'text/plain' in case of using 'aws s3 cp' (instead of aws s3api).
+ * <p>
+ * Our generic ObjectEndpoint.multipartUpload has a
+ * CompleteMultipartUploadRequest parameter, which is required only for the
+ * completion request.
+ * <p>
+ * But JaxRS tries to parse it from the body for the requests and in case of
+ * text/plain requests this parsing is failed. This simple BodyReader enables
+ * to parse an empty text/plain message and return with an empty completion
+ * request.
+ */
+@Provider
+@Consumes("text/plain")
+public class PlainTextMultipartUploadReader
+ implements MessageBodyReader<CompleteMultipartUploadRequest> {
+
+ @Override
+ public boolean isReadable(Class<?> type, Type genericType,
+ Annotation[] annotations, MediaType mediaType) {
+ return type.equals(CompleteMultipartUploadRequest.class)
+ && mediaType.equals(MediaType.TEXT_PLAIN_TYPE);
+ }
+
+ @Override
+ public CompleteMultipartUploadRequest readFrom(
+ Class<CompleteMultipartUploadRequest> type, Type genericType,
+ Annotation[] annotations, MediaType mediaType,
+ MultivaluedMap<String, String> httpHeaders, InputStream entityStream)
+ throws IOException, WebApplicationException {
+ return new CompleteMultipartUploadRequest();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org