You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2018/10/31 12:33:20 UTC
[1/2] hadoop git commit: HDDS-712. Use x-amz-storage-class to specify
replication type and replication factor. Contributed by Bharat Viswanadham.
Repository: hadoop
Updated Branches:
refs/heads/trunk 9c438abe5 -> b519f3f2a
HDDS-712. Use x-amz-storage-class to specify replication type and replication factor. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ecac351a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ecac351a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ecac351a
Branch: refs/heads/trunk
Commit: ecac351aac1702194c56743ced5a66242643f28c
Parents: 9c438ab
Author: Márton Elek <el...@apache.org>
Authored: Wed Oct 31 11:08:43 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Wed Oct 31 13:28:59 2018 +0100
----------------------------------------------------------------------
.../dist/src/main/smoketest/s3/awss3.robot | 4 +-
.../dist/src/main/smoketest/s3/objectcopy.robot | 18 ++---
.../src/main/smoketest/s3/objectdelete.robot | 6 +-
.../main/smoketest/s3/objectmultidelete.robot | 6 +-
.../src/main/smoketest/s3/objectputget.robot | 2 +-
.../ozone/s3/endpoint/ObjectEndpoint.java | 68 ++++++++++++++-----
.../apache/hadoop/ozone/s3/util/S3Consts.java | 19 ++++++
.../hadoop/ozone/s3/util/S3StorageType.java | 55 ++++++++++++++++
.../hadoop/ozone/s3/util/package-info.java | 22 +++++++
.../hadoop/ozone/s3/endpoint/TestPutObject.java | 69 +++++++++++---------
10 files changed, 205 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/dist/src/main/smoketest/s3/awss3.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/awss3.robot b/hadoop-ozone/dist/src/main/smoketest/s3/awss3.robot
index 79db688..c1ec9f0 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/awss3.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/awss3.robot
@@ -29,9 +29,9 @@ ${BUCKET} generated
File upload and directory list
Execute date > /tmp/testfile
- ${result} = Execute AWSS3Cli cp /tmp/testfile s3://${BUCKET}
+ ${result} = Execute AWSS3Cli cp --storage-class REDUCED_REDUNDANCY /tmp/testfile s3://${BUCKET}
Should contain ${result} upload
- ${result} = Execute AWSS3Cli cp /tmp/testfile s3://${BUCKET}/dir1/dir2/file
+ ${result} = Execute AWSS3Cli cp --storage-class REDUCED_REDUNDANCY /tmp/testfile s3://${BUCKET}/dir1/dir2/file
Should contain ${result} upload
${result} = Execute AWSS3Cli ls s3://${BUCKET}
Should contain ${result} testfile
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
index 2daa861..e702d9b 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
@@ -39,28 +39,28 @@ Create Dest Bucket
Copy Object Happy Scenario
Run Keyword if '${DESTBUCKET}' == 'generated1' Create Dest Bucket
Execute date > /tmp/copyfile
- ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key copyobject/f1 --body /tmp/copyfile
+ ${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key copyobject/f1 --body /tmp/copyfile
${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix copyobject/
Should contain ${result} f1
- ${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1
+ ${result} = Execute AWSS3ApiCli copy-object --storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1
${result} = Execute AWSS3ApiCli list-objects --bucket ${DESTBUCKET} --prefix copyobject/
Should contain ${result} f1
#copying again will not throw error
- ${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1
+ ${result} = Execute AWSS3ApiCli copy-object --storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1
${result} = Execute AWSS3ApiCli list-objects --bucket ${DESTBUCKET} --prefix copyobject/
Should contain ${result} f1
Copy Object Where Bucket is not available
- ${result} = Execute AWSS3APICli and checkrc copy-object --bucket dfdfdfdfdfnonexistent --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1 255
+ ${result} = Execute AWSS3APICli and checkrc copy-object --storage-class REDUCED_REDUNDANCY --bucket dfdfdfdfdfnonexistent --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1 255
Should contain ${result} NoSuchBucket
- ${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source dfdfdfdfdfnonexistent/copyobject/f1 255
+ ${result} = Execute AWSS3APICli and checkrc copy-object --storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source dfdfdfdfdfnonexistent/copyobject/f1 255
Should contain ${result} NoSuchBucket
-Copy Object Where both source and dest are same
- ${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${DESTBUCKET}/copyobject/f1 255
- Should contain ${result} InvalidRequest
+Copy Object Where both source and dest are same with change to storageclass
+ ${result} = Execute AWSS3APICli copy-object --storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${DESTBUCKET}/copyobject/f1
+ Should contain ${result} ETag
Copy Object Where Key not available
- ${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/nonnonexistentkey 255
+ ${result} = Execute AWSS3APICli and checkrc copy-object --storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/nonnonexistentkey 255
Should contain ${result} NoSuchKey
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
index 9e57d50..44e1a22 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
@@ -28,7 +28,7 @@ ${BUCKET} generated
*** Test Cases ***
Delete file with s3api
Execute date > /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key deletetestapi/f1 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key deletetestapi/f1 --body /tmp/testfile
${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix deletetestapi/
Should contain ${result} f1
${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key deletetestapi/f1
@@ -46,7 +46,7 @@ Delete file with s3api, file doesn't exist
Delete dir with s3api
Execute date > /tmp/testfile
- ${result} = Execute AWSS3Cli cp /tmp/testfile s3://${BUCKET}/deletetestapidir/f1
+ ${result} = Execute AWSS3Cli cp --storage-class REDUCED_REDUNDANCY /tmp/testfile s3://${BUCKET}/deletetestapidir/f1
${result} = Execute AWSS3Cli ls s3://${BUCKET}/deletetestapidir/
Should contain ${result} f1
${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key deletetestapidir/
@@ -57,7 +57,7 @@ Delete dir with s3api
Delete file with s3api, file doesn't exist, prefix of a real file
Execute date > /tmp/testfile
- ${result} = Execute AWSS3Cli cp /tmp/testfile s3://${BUCKET}/deletetestapiprefix/filefile
+ ${result} = Execute AWSS3Cli cp --storage-class REDUCED_REDUNDANCY /tmp/testfile s3://${BUCKET}/deletetestapiprefix/filefile
${result} = Execute AWSS3Cli ls s3://${BUCKET}/deletetestapiprefix/
Should contain ${result} filefile
${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key deletetestapiprefix/file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
index 83c967b..e8b7bd3 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
@@ -29,9 +29,9 @@ ${BUCKET} generated
Delete file with multi delete
Execute date > /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key multidelete/f1 --body /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key multidelete/f2 --body /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key multidelete/f3 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key multidelete/f1 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key multidelete/f2 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key multidelete/f3 --body /tmp/testfile
${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix multidelete/
Should contain ${result} multidelete/f1
Should contain ${result} multidelete/f2
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
index 858e472..f650de8 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
@@ -30,7 +30,7 @@ ${BUCKET} generated
Put object to s3
Execute date > /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key putobject/f1 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key putobject/f1 --body /tmp/testfile
${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix putobject/
Should contain ${result} f1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index 9782d75..7000a3a 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -18,14 +18,12 @@
package org.apache.hadoop.ozone.s3.endpoint;
import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.HEAD;
import javax.ws.rs.HeaderParam;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
-import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
@@ -53,11 +51,16 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.ozone.s3.util.S3StorageType;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.util.Time;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+
/**
* Key level rest endpoints.
*/
@@ -91,22 +94,37 @@ public class ObjectEndpoint extends EndpointBase {
public Response put(
@PathParam("bucket") String bucketName,
@PathParam("path") String keyPath,
- @DefaultValue("STAND_ALONE") @QueryParam("replicationType")
- ReplicationType replicationType,
- @DefaultValue("ONE") @QueryParam("replicationFactor")
- ReplicationFactor replicationFactor,
@HeaderParam("Content-Length") long length,
InputStream body) throws IOException, OS3Exception {
OzoneOutputStream output = null;
try {
- String copyHeader = headers.getHeaderString("x-amz-copy-source");
+ String copyHeader = headers.getHeaderString(COPY_SOURCE_HEADER);
+ String storageType = headers.getHeaderString(STORAGE_CLASS_HEADER);
+
+ ReplicationType replicationType;
+ ReplicationFactor replicationFactor;
+ boolean storageTypeDefault;
+ if (storageType == null) {
+ replicationType = S3StorageType.getDefault().getType();
+ replicationFactor = S3StorageType.getDefault().getFactor();
+ storageTypeDefault = true;
+ } else {
+ try {
+ replicationType = S3StorageType.valueOf(storageType).getType();
+ replicationFactor = S3StorageType.valueOf(storageType).getFactor();
+ } catch (IllegalArgumentException ex) {
+ throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT,
+ storageType);
+ }
+ storageTypeDefault = false;
+ }
if (copyHeader != null) {
//Copy object, as copy source available.
CopyObjectResponse copyObjectResponse = copyObject(
copyHeader, bucketName, keyPath, replicationType,
- replicationFactor);
+ replicationFactor, storageTypeDefault);
return Response.status(Status.OK).entity(copyObjectResponse).header(
"Connection", "close").build();
}
@@ -254,7 +272,8 @@ public class ObjectEndpoint extends EndpointBase {
String destBucket,
String destkey,
ReplicationType replicationType,
- ReplicationFactor replicationFactor)
+ ReplicationFactor replicationFactor,
+ boolean storageTypeDefault)
throws OS3Exception, IOException {
if (copyHeader.startsWith("/")) {
@@ -276,18 +295,35 @@ public class ObjectEndpoint extends EndpointBase {
boolean closed = false;
try {
// Checking whether we trying to copying to it self.
+
if (sourceBucket.equals(destBucket)) {
if (sourceKey.equals(destkey)) {
- OS3Exception ex = S3ErrorTable.newError(S3ErrorTable
- .INVALID_REQUEST, copyHeader);
- ex.setErrorMessage("This copy request is illegal because it is " +
- "trying to copy an object to it self itself without changing " +
- "the object's metadata, storage class, website redirect " +
- "location or encryption attributes.");
- throw ex;
+ // When copying to same storage type when storage type is provided,
+ // we should not throw exception, as aws cli checks if any of the
+ // options like storage type are provided or not when source and
+ // dest are given same
+ if (storageTypeDefault) {
+ OS3Exception ex = S3ErrorTable.newError(S3ErrorTable
+ .INVALID_REQUEST, copyHeader);
+ ex.setErrorMessage("This copy request is illegal because it is " +
+ "trying to copy an object to it self itself without changing " +
+ "the object's metadata, storage class, website redirect " +
+ "location or encryption attributes.");
+ throw ex;
+ } else {
+ // TODO: Actually here we should change storage type, as ozone
+ // still does not support this just returning dummy response
+ // for now
+ CopyObjectResponse copyObjectResponse = new CopyObjectResponse();
+ copyObjectResponse.setETag(OzoneUtils.getRequestID());
+ copyObjectResponse.setLastModified(Instant.ofEpochMilli(
+ Time.now()));
+ return copyObjectResponse;
+ }
}
}
+
OzoneBucket sourceOzoneBucket = getBucket(sourceBucket);
OzoneBucket destOzoneBucket = getBucket(destBucket);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
new file mode 100644
index 0000000..2e7b965
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -0,0 +1,19 @@
+package org.apache.hadoop.ozone.s3.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Set of constants used for S3 implementation.
+ */
+@InterfaceAudience.Private
+public final class S3Consts {
+
+ //Never Constructed
+ private S3Consts() {
+
+ }
+
+ public static final String COPY_SOURCE_HEADER = "x-amz-copy-source";
+ public static final String STORAGE_CLASS_HEADER = "x-amz-storage-class";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3StorageType.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3StorageType.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3StorageType.java
new file mode 100644
index 0000000..c9d1b50
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3StorageType.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.util;
+
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+
+/**
+ * Maps S3 storage class values to Ozone replication values.
+ */
+
+public enum S3StorageType {
+
+ REDUCED_REDUNDANCY(ReplicationType.STAND_ALONE, ReplicationFactor.ONE),
+ STANDARD(ReplicationType.RATIS, ReplicationFactor.THREE);
+
+ private final ReplicationType type;
+ private final ReplicationFactor factor;
+
+ S3StorageType(
+ ReplicationType type,
+ ReplicationFactor factor) {
+ this.type = type;
+ this.factor = factor;
+ }
+
+ public ReplicationFactor getFactor() {
+ return factor;
+ }
+
+ public ReplicationType getType() {
+ return type;
+ }
+
+ public static S3StorageType getDefault() {
+ return STANDARD;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/package-info.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/package-info.java
new file mode 100644
index 0000000..af93f08
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains Ozone S3 Util classes.
+ */
+package org.apache.hadoop.ozone.s3.util;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java
index 4f94e56..eb05861 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java
@@ -26,19 +26,21 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.charset.Charset;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.when;
@@ -80,9 +82,9 @@ public class TestPutObject {
objectEndpoint.setHeaders(headers);
//WHEN
- Response response = objectEndpoint.put(bucketName, keyName,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
- body);
+ Response response = objectEndpoint.put(bucketName, keyName, CONTENT
+ .length(), body);
+
//THEN
String volumeName = clientStub.getObjectStore()
@@ -113,8 +115,6 @@ public class TestPutObject {
//WHEN
Response response = objectEndpoint.put(bucketName, keyName,
- ReplicationType.STAND_ALONE,
- ReplicationFactor.ONE,
chunkedContent.length(),
new ByteArrayInputStream(chunkedContent.getBytes()));
@@ -140,8 +140,7 @@ public class TestPutObject {
keyName = "sourceKey";
Response response = objectEndpoint.put(bucketName, keyName,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
- body);
+ CONTENT.length(), body);
String volumeName = clientStub.getObjectStore().getOzoneVolumeName(
bucketName);
@@ -157,12 +156,10 @@ public class TestPutObject {
// Add copy header, and then call put
- when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
bucketName + "/" + keyName);
- response = objectEndpoint.put(destBucket, destkey,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
- body);
+ response = objectEndpoint.put(destBucket, destkey, CONTENT.length(), body);
// Check destination key and response
volumeName = clientStub.getObjectStore().getOzoneVolumeName(destBucket);
@@ -176,8 +173,7 @@ public class TestPutObject {
// source and dest same
try {
- objectEndpoint.put(bucketName, keyName, ReplicationType.STAND_ALONE,
- ReplicationFactor.ONE, CONTENT.length(), body);
+ objectEndpoint.put(bucketName, keyName, CONTENT.length(), body);
fail("test copy object failed");
} catch (OS3Exception ex) {
Assert.assertTrue(ex.getErrorMessage().contains("This copy request is " +
@@ -186,10 +182,9 @@ public class TestPutObject {
// source bucket not found
try {
- when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
nonexist + "/" + keyName);
- response = objectEndpoint.put(destBucket, destkey,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
+ objectEndpoint.put(destBucket, destkey, CONTENT.length(),
body);
fail("test copy object failed");
} catch (OS3Exception ex) {
@@ -198,11 +193,9 @@ public class TestPutObject {
// dest bucket not found
try {
- when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
bucketName + "/" + keyName);
- response = objectEndpoint.put(nonexist, destkey,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
- body);
+ objectEndpoint.put(nonexist, destkey, CONTENT.length(), body);
fail("test copy object failed");
} catch (OS3Exception ex) {
Assert.assertTrue(ex.getCode().contains("NoSuchBucket"));
@@ -210,11 +203,9 @@ public class TestPutObject {
//Both source and dest bucket not found
try {
- when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
nonexist + "/" + keyName);
- response = objectEndpoint.put(nonexist, destkey,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
- body);
+ objectEndpoint.put(nonexist, destkey, CONTENT.length(), body);
fail("test copy object failed");
} catch (OS3Exception ex) {
Assert.assertTrue(ex.getCode().contains("NoSuchBucket"));
@@ -222,15 +213,33 @@ public class TestPutObject {
// source key not found
try {
- when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
bucketName + "/" + nonexist);
- response = objectEndpoint.put("nonexistent", keyName,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
- body);
+ objectEndpoint.put("nonexistent", keyName, CONTENT.length(), body);
fail("test copy object failed");
} catch (OS3Exception ex) {
Assert.assertTrue(ex.getCode().contains("NoSuchBucket"));
}
}
+
+ @Test
+ public void testInvalidStorageType() throws IOException {
+ HttpHeaders headers = Mockito.mock(HttpHeaders.class);
+ ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes());
+ objectEndpoint.setHeaders(headers);
+ keyName = "sourceKey";
+ when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("random");
+
+ try {
+ Response response = objectEndpoint.put(bucketName, keyName,
+ CONTENT.length(), body);
+ fail("testInvalidStorageType");
+ } catch (OS3Exception ex) {
+ assertEquals(S3ErrorTable.INVALID_ARGUMENT.getErrorMessage(),
+ ex.getErrorMessage());
+ assertEquals("random", ex.getResource());
+ }
+
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: HDDS-659. Implement pagination in GET bucket
(object list) endpoint. Contributed by Bharat Viswanadham.
Posted by el...@apache.org.
HDDS-659. Implement pagination in GET bucket (object list) endpoint. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b519f3f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b519f3f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b519f3f2
Branch: refs/heads/trunk
Commit: b519f3f2a0ae960391ce7bff59f1fdd21a22e030
Parents: ecac351
Author: Márton Elek <el...@apache.org>
Authored: Wed Oct 31 12:21:38 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Wed Oct 31 13:29:01 2018 +0100
----------------------------------------------------------------------
.../ozone/s3/endpoint/BucketEndpoint.java | 109 ++++++---
.../ozone/s3/endpoint/ListObjectResponse.java | 22 ++
.../apache/hadoop/ozone/s3/util/S3Consts.java | 1 +
.../apache/hadoop/ozone/s3/util/S3utils.java | 73 ++++++
.../hadoop/ozone/client/OzoneBucketStub.java | 7 +-
.../hadoop/ozone/s3/endpoint/TestBucketGet.java | 227 ++++++++++++++++++-
6 files changed, 400 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b519f3f2/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index 8f554ed..04e2348 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -36,7 +36,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.time.Instant;
import java.util.Iterator;
-import javax.ws.rs.core.Response.ResponseBuilder;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneKey;
@@ -48,10 +47,13 @@ import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.s3.util.S3utils;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE;
+
/**
* Bucket level rest endpoints.
*/
@@ -76,6 +78,8 @@ public class BucketEndpoint extends EndpointBase {
@DefaultValue("1000") @QueryParam("max-keys") int maxKeys,
@QueryParam("prefix") String prefix,
@QueryParam("browser") String browser,
+ @QueryParam("continuation-token") String continueToken,
+ @QueryParam("start-after") String startAfter,
@Context HttpHeaders hh) throws OS3Exception, IOException {
if (browser != null) {
@@ -87,60 +91,91 @@ public class BucketEndpoint extends EndpointBase {
}
}
- if (delimiter == null) {
- delimiter = "/";
- }
if (prefix == null) {
prefix = "";
}
OzoneBucket bucket = getBucket(bucketName);
- Iterator<? extends OzoneKey> ozoneKeyIterator = bucket.listKeys(prefix);
+ Iterator<? extends OzoneKey> ozoneKeyIterator;
+
+ String decodedToken = S3utils.decodeContinueToken(continueToken);
+
+ if (startAfter != null && continueToken != null) {
+ // If continuation token and start after both are provided, then we
+ // ignore start After
+ ozoneKeyIterator = bucket.listKeys(prefix, decodedToken);
+ } else if (startAfter != null && continueToken == null) {
+ ozoneKeyIterator = bucket.listKeys(prefix, startAfter);
+ } else if (startAfter == null && continueToken != null){
+ ozoneKeyIterator = bucket.listKeys(prefix, decodedToken);
+ } else {
+ ozoneKeyIterator = bucket.listKeys(prefix);
+ }
+
ListObjectResponse response = new ListObjectResponse();
response.setDelimiter(delimiter);
response.setName(bucketName);
response.setPrefix(prefix);
response.setMarker("");
- response.setMaxKeys(1000);
- response.setEncodingType("url");
+ response.setMaxKeys(maxKeys);
+ response.setEncodingType(ENCODING_TYPE);
response.setTruncated(false);
+ response.setContinueToken(continueToken);
String prevDir = null;
+ String lastKey = null;
+ int count = 0;
while (ozoneKeyIterator.hasNext()) {
OzoneKey next = ozoneKeyIterator.next();
String relativeKeyName = next.getName().substring(prefix.length());
- int depth =
- StringUtils.countMatches(relativeKeyName, delimiter);
+ int depth = StringUtils.countMatches(relativeKeyName, delimiter);
+ if (delimiter != null) {
+ if (depth > 0) {
+ // means key has multiple delimiters in its value.
+ // ex: dir/dir1/dir2, where delimiter is "/" and prefix is dir/
+ String dirName = relativeKeyName.substring(0, relativeKeyName
+ .indexOf(delimiter));
+ if (!dirName.equals(prevDir)) {
+ response.addPrefix(prefix + dirName + delimiter);
+ prevDir = dirName;
+ count++;
+ }
+ } else if (relativeKeyName.endsWith(delimiter)) {
+ // means or key is same as prefix with delimiter at end and ends with
+ // delimiter. ex: dir/, where prefix is dir and delimiter is /
+ response.addPrefix(relativeKeyName);
+ count++;
+ } else {
+ // means our key is matched with prefix if prefix is given and it
+ // does not have any common prefix.
+ addKey(response, next);
+ count++;
+ }
+ } else {
+ addKey(response, next);
+ count++;
+ }
- if (prefix.length() > 0 && !prefix.endsWith(delimiter)
- && relativeKeyName.length() > 0) {
- response.addPrefix(prefix + "/");
+ if (count == maxKeys) {
+ lastKey = next.getName();
break;
}
- if (depth > 0) {
- String dirName = relativeKeyName
- .substring(0, relativeKeyName.indexOf(delimiter));
- if (!dirName.equals(prevDir)) {
- response.addPrefix(
- prefix + dirName + delimiter);
- prevDir = dirName;
- }
- } else if (relativeKeyName.endsWith(delimiter)) {
- response.addPrefix(relativeKeyName);
- } else if (relativeKeyName.length() > 0) {
- KeyMetadata keyMetadata = new KeyMetadata();
- keyMetadata.setKey(next.getName());
- keyMetadata.setSize(next.getDataSize());
- keyMetadata.setETag("" + next.getModificationTime());
- keyMetadata.setStorageClass("STANDARD");
- keyMetadata
- .setLastModified(Instant.ofEpochMilli(next.getModificationTime()));
- response.addKey(keyMetadata);
- }
}
+
+ response.setKeyCount(count);
+
+ if (count < maxKeys) {
+ response.setTruncated(false);
+ } else if(ozoneKeyIterator.hasNext()) {
+ response.setTruncated(true);
+ response.setNextToken(S3utils.generateContinueToken(lastKey));
+ } else {
+ response.setTruncated(false);
+ }
+
response.setKeyCount(
response.getCommonPrefixes().size() + response.getContents().size());
return Response.ok(response).build();
@@ -253,4 +288,14 @@ public class BucketEndpoint extends EndpointBase {
}
return result;
}
+
+ private void addKey(ListObjectResponse response, OzoneKey next) {
+ KeyMetadata keyMetadata = new KeyMetadata();
+ keyMetadata.setKey(next.getName());
+ keyMetadata.setSize(next.getDataSize());
+ keyMetadata.setETag("" + next.getModificationTime());
+ keyMetadata.setStorageClass("STANDARD");
+ keyMetadata.setLastModified(Instant.ofEpochMilli(next.getModificationTime()));
+ response.addKey(keyMetadata);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b519f3f2/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListObjectResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListObjectResponse.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListObjectResponse.java
index b9ab977..adb5f20 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListObjectResponse.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListObjectResponse.java
@@ -60,6 +60,12 @@ public class ListObjectResponse {
@XmlElement(name = "IsTruncated")
private boolean isTruncated;
+ @XmlElement(name = "NextContinuationToken")
+ private String nextToken;
+
+ @XmlElement(name = "continueToken")
+ private String continueToken;
+
@XmlElement(name = "Contents")
private List<KeyMetadata> contents = new ArrayList<>();
@@ -148,6 +154,22 @@ public class ListObjectResponse {
commonPrefixes.add(new CommonPrefix(relativeKeyName));
}
+ public String getNextToken() {
+ return nextToken;
+ }
+
+ public void setNextToken(String nextToken) {
+ this.nextToken = nextToken;
+ }
+
+ public String getContinueToken() {
+ return continueToken;
+ }
+
+ public void setContinueToken(String continueToken) {
+ this.continueToken = continueToken;
+ }
+
public int getKeyCount() {
return keyCount;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b519f3f2/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index 2e7b965..70d8a96 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -15,5 +15,6 @@ public final class S3Consts {
public static final String COPY_SOURCE_HEADER = "x-amz-copy-source";
public static final String STORAGE_CLASS_HEADER = "x-amz-storage-class";
+ public static final String ENCODING_TYPE = "url";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b519f3f2/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
new file mode 100644
index 0000000..8af0927
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
@@ -0,0 +1,73 @@
+package org.apache.hadoop.ozone.s3.util;
+
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Utility class for S3.
+ */
+public final class S3utils {
+
+ private S3utils() {
+
+ }
+ private static final String CONTINUE_TOKEN_SEPERATOR = "-";
+
+ /**
+ * Generate a continuation token which is used in get Bucket.
+ * @param key
+ * @return if key is not null return continuation token, else returns null.
+ */
+ public static String generateContinueToken(String key) {
+ if (key != null) {
+ byte[] byteData = key.getBytes(StandardCharsets.UTF_8);
+ String hex = Hex.encodeHexString(byteData);
+ String digest = DigestUtils.sha256Hex(key);
+ return hex + CONTINUE_TOKEN_SEPERATOR + digest;
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Decode a continuation token which is used in get Bucket.
+ * @param key
+ * @return if key is not null return decoded token, otherwise returns null.
+ * @throws OS3Exception
+ */
+ public static String decodeContinueToken(String key) throws OS3Exception {
+ if (key != null) {
+ int indexSeparator = key.indexOf(CONTINUE_TOKEN_SEPERATOR);
+ if (indexSeparator == -1) {
+ throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, key);
+ }
+ String hex = key.substring(0, indexSeparator);
+ String digest = key.substring(indexSeparator + 1);
+ try {
+ byte[] actualKeyBytes = Hex.decodeHex(hex);
+ String digestActualKey = DigestUtils.sha256Hex(actualKeyBytes);
+ if (digest.equals(digestActualKey)) {
+ return new String(actualKeyBytes);
+ } else {
+ OS3Exception ex = S3ErrorTable.newError(S3ErrorTable
+ .INVALID_ARGUMENT, key);
+ ex.setErrorMessage("The continuation token provided is incorrect");
+ throw ex;
+ }
+ } catch (DecoderException ex) {
+ OS3Exception os3Exception = S3ErrorTable.newError(S3ErrorTable
+ .INVALID_ARGUMENT, key);
+ os3Exception.setErrorMessage("The continuation token provided is " +
+ "incorrect");
+ throw os3Exception;
+ }
+ } else {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b519f3f2/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
index 6aff087..918e9d0 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
@@ -27,6 +27,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.client.ReplicationFactor;
@@ -116,7 +117,8 @@ public class OzoneBucketStub extends OzoneBucket {
@Override
public Iterator<? extends OzoneKey> listKeys(String keyPrefix) {
- return keyDetails.values()
+ Map<String, OzoneKey> sortedKey = new TreeMap<String, OzoneKey>(keyDetails);
+ return sortedKey.values()
.stream()
.filter(key -> key.getName().startsWith(keyPrefix))
.collect(Collectors.toList())
@@ -126,7 +128,8 @@ public class OzoneBucketStub extends OzoneBucket {
@Override
public Iterator<? extends OzoneKey> listKeys(String keyPrefix,
String prevKey) {
- return keyDetails.values()
+ Map<String, OzoneKey> sortedKey = new TreeMap<String, OzoneKey>(keyDetails);
+ return sortedKey.values()
.stream()
.filter(key -> key.getName().compareTo(prevKey) > 0)
.filter(key -> key.getName().startsWith(keyPrefix))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b519f3f2/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java
index 54534ed..8b70d29 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.junit.Assert;
import org.junit.Test;
+import static org.junit.Assert.fail;
+
/**
* Testing basic object list browsing.
*/
@@ -45,7 +47,7 @@ public class TestBucketGet {
ListObjectResponse getBucketResponse =
(ListObjectResponse) getBucket
- .list("b1", "/", null, null, 100, "", null, null)
+ .list("b1", "/", null, null, 100, "", null, null, null, null)
.getEntity();
Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size());
@@ -68,8 +70,8 @@ public class TestBucketGet {
getBucket.setClient(client);
ListObjectResponse getBucketResponse =
- (ListObjectResponse) getBucket
- .list("b1", "/", null, null, 100, "dir1", null, null).getEntity();
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, 100,
+ "dir1", null, null, null, null).getEntity();
Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size());
Assert.assertEquals("dir1/",
@@ -85,13 +87,16 @@ public class TestBucketGet {
BucketEndpoint getBucket = new BucketEndpoint();
OzoneClient ozoneClient =
- createClientWithKeys("dir1/file2", "dir1/dir2/file2");
+ createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file",
+ "dir1bha/file2");
getBucket.setClient(ozoneClient);
ListObjectResponse getBucketResponse =
(ListObjectResponse) getBucket
- .list("b1", "/", null, null, 100, "dir1/", null, null).getEntity();
+ .list("b1", "/", null, null, 100, "dir1/", null, null,
+ null, null)
+ .getEntity();
Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size());
Assert.assertEquals("dir1/dir2/",
@@ -103,6 +108,218 @@ public class TestBucketGet {
}
+
+ @Test
+ public void listWithPrefixAndDelimiter() throws OS3Exception, IOException {
+
+ BucketEndpoint getBucket = new BucketEndpoint();
+
+ OzoneClient ozoneClient =
+ createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file",
+ "dir1bha/file2", "file2");
+
+ getBucket.setClient(ozoneClient);
+
+ ListObjectResponse getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, 100,
+ "dir1", null, null, null, null).getEntity();
+
+ Assert.assertEquals(3, getBucketResponse.getCommonPrefixes().size());
+
+ }
+
+ @Test
+ public void listWithPrefixAndDelimiter1() throws OS3Exception, IOException {
+
+ BucketEndpoint getBucket = new BucketEndpoint();
+
+ OzoneClient ozoneClient =
+ createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file",
+ "dir1bha/file2", "file2");
+
+ getBucket.setClient(ozoneClient);
+
+ ListObjectResponse getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, 100,
+ "", null, null, null, null).getEntity();
+
+ Assert.assertEquals(3, getBucketResponse.getCommonPrefixes().size());
+ Assert.assertEquals("file2", getBucketResponse.getContents().get(0)
+ .getKey());
+
+ }
+
+ @Test
+ public void listWithPrefixAndDelimiter2() throws OS3Exception, IOException {
+
+ BucketEndpoint getBucket = new BucketEndpoint();
+
+ OzoneClient ozoneClient =
+ createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file",
+ "dir1bha/file2", "file2");
+
+ getBucket.setClient(ozoneClient);
+
+ ListObjectResponse getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, 100,
+ "dir1bh", null, null, "dir1/dir2/file2", null).getEntity();
+
+ Assert.assertEquals(2, getBucketResponse.getCommonPrefixes().size());
+
+ }
+
+ @Test
+ public void listWithContinuationToken() throws OS3Exception, IOException {
+
+ BucketEndpoint getBucket = new BucketEndpoint();
+
+ OzoneClient ozoneClient =
+ createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file",
+ "dir1bha/file2", "file2");
+
+ getBucket.setClient(ozoneClient);
+
+ int maxKeys = 2;
+ // As we have 5 keys, with max keys 2 we should call list 3 times.
+
+ // First time
+ ListObjectResponse getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", null, null, null, maxKeys,
+ "", null, null, null, null).getEntity();
+
+ Assert.assertTrue(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getContents().size() == 2);
+
+ // 2nd time
+ String continueToken = getBucketResponse.getNextToken();
+ getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", null, null, null, maxKeys,
+ "", null, continueToken, null, null).getEntity();
+ Assert.assertTrue(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getContents().size() == 2);
+
+
+ continueToken = getBucketResponse.getNextToken();
+
+ //3rd time
+ getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", null, null, null, maxKeys,
+ "", null, continueToken, null, null).getEntity();
+
+ Assert.assertFalse(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getContents().size() == 1);
+
+ }
+
+ @Test
+ /**
+ * This test is with prefix and delimiter and verify continuation-token
+ * behavior.
+ */
+ public void listWithContinuationToken1() throws OS3Exception, IOException {
+
+ BucketEndpoint getBucket = new BucketEndpoint();
+
+ OzoneClient ozoneClient =
+ createClientWithKeys("dir1/file1", "dir1bh/file1",
+ "dir1bha/file1", "dir0/file1", "dir2/file1");
+
+ getBucket.setClient(ozoneClient);
+
+ int maxKeys = 2;
+ // As we have 5 keys, with max keys 2 we should call list 3 times.
+
+ // First time
+ ListObjectResponse getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, maxKeys,
+ "dir", null, null, null, null).getEntity();
+
+ Assert.assertTrue(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getCommonPrefixes().size() == 2);
+
+
+ // 2nd time
+ String continueToken = getBucketResponse.getNextToken();
+ getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, maxKeys,
+ "dir", null, continueToken, null, null).getEntity();
+ Assert.assertTrue(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getCommonPrefixes().size() == 2);
+
+
+ //3rd time
+ continueToken = getBucketResponse.getNextToken();
+ getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, maxKeys,
+ "dir", null, continueToken, null, null).getEntity();
+
+ Assert.assertFalse(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getCommonPrefixes().size() == 1);
+
+ }
+
+ @Test
+ public void listWithContinuationTokenFail() throws OS3Exception, IOException {
+
+ BucketEndpoint getBucket = new BucketEndpoint();
+
+ OzoneClient ozoneClient =
+ createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file",
+ "dir1bha/file2", "dir1", "dir2", "dir3");
+
+ getBucket.setClient(ozoneClient);
+
+ try {
+ ListObjectResponse getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, 2,
+ "dir", null, "random", null, null).getEntity();
+ fail("listWithContinuationTokenFail");
+ } catch (OS3Exception ex) {
+ Assert.assertEquals("random", ex.getResource());
+ Assert.assertEquals("Invalid Argument", ex.getErrorMessage());
+ }
+
+ }
+
+
+ @Test
+ public void testStartAfter() throws IOException, OS3Exception {
+ BucketEndpoint getBucket = new BucketEndpoint();
+
+ OzoneClient ozoneClient =
+ createClientWithKeys("dir1/file1", "dir1bh/file1",
+ "dir1bha/file1", "dir0/file1", "dir2/file1");
+
+ getBucket.setClient(ozoneClient);
+
+ ListObjectResponse getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", null, null, null, 1000,
+ null, null, null, null, null).getEntity();
+
+ Assert.assertFalse(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getContents().size() == 5);
+
+ //As our list output is sorted, after seeking to startAfter, we shall
+ // have 4 keys.
+ String startAfter = "dir0/file1";
+
+ getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", null, null, null,
+ 1000, null, null, null, startAfter, null).getEntity();
+
+ Assert.assertFalse(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getContents().size() == 4);
+
+ getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", null, null, null,
+ 1000, null, null, null, "random", null).getEntity();
+
+ Assert.assertFalse(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getContents().size() == 0);
+
+
+ }
+
private OzoneClient createClientWithKeys(String... keys) throws IOException {
OzoneClient client = new OzoneClientStub();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org