You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2018/11/15 11:22:38 UTC
hadoop git commit: HDDS-658. Implement s3 bucket list backend call
and use it from rest endpoint. Contributed by Bharat Viswanadham.
Repository: hadoop
Updated Branches:
refs/heads/trunk 9b01f039e -> b12e69475
HDDS-658. Implement s3 bucket list backend call and use it from rest endpoint. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b12e6947
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b12e6947
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b12e6947
Branch: refs/heads/trunk
Commit: b12e69475bffb90b82cb7a905c96738b3fd91a7b
Parents: 9b01f03
Author: Márton Elek <el...@apache.org>
Authored: Thu Nov 15 12:11:05 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Thu Nov 15 12:22:16 2018 +0100
----------------------------------------------------------------------
.../org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../apache/hadoop/ozone/client/ObjectStore.java | 113 ++++++++++++++++++-
.../ozone/client/protocol/ClientProtocol.java | 16 +++
.../hadoop/ozone/client/rest/RestClient.java | 8 ++
.../hadoop/ozone/client/rpc/RpcClient.java | 19 ++++
.../org/apache/hadoop/ozone/audit/OMAction.java | 3 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 23 ++++
...neManagerProtocolClientSideTranslatorPB.java | 38 +++++++
.../src/main/proto/OzoneManagerProtocol.proto | 15 +++
.../ozone/client/rpc/TestOzoneRpcClient.java | 34 ++++++
.../org/apache/hadoop/ozone/om/OMMetrics.java | 23 ++++
.../apache/hadoop/ozone/om/OzoneManager.java | 30 +++++
.../apache/hadoop/ozone/om/S3BucketManager.java | 6 +
.../hadoop/ozone/om/S3BucketManagerImpl.java | 10 +-
...neManagerProtocolServerSideTranslatorPB.java | 24 ++++
.../hadoop/ozone/om/TestS3BucketManager.java | 25 ++++
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 32 ++++++
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 24 ++--
.../hadoop/ozone/client/ObjectStoreStub.java | 72 +++++++++++-
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 1 -
.../hadoop/ozone/s3/endpoint/TestRootList.java | 11 +-
21 files changed, 489 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index b77d621..d37ce0f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -144,6 +144,8 @@ public final class OzoneConsts {
public static final String OM_KEY_PREFIX = "/";
public static final String OM_USER_PREFIX = "$";
public static final String OM_S3_PREFIX ="S3:";
+ public static final String OM_S3_VOLUME_PREFIX = "s3";
+
/**
* Max OM Quota size of 1024 PB.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index 1ea5c03..136a684 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -18,17 +18,20 @@
package org.apache.hadoop.ozone.client;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Objects;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.security.UserGroupInformation;
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
/**
* ObjectStore class is responsible for the client operations that can be
@@ -150,6 +153,36 @@ public class ObjectStore {
return volume;
}
+ /**
+ * Returns Iterator to iterate over all buckets for a user.
+ * The result can be restricted using bucket prefix, will return all
+ * buckets if bucket prefix is null.
+ *
+ * @param userName user name
+ * @param bucketPrefix Bucket prefix to match
+ * @return {@code Iterator<OzoneBucket>}
+ */
+ public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
+ String bucketPrefix) {
+ return listS3Buckets(userName, bucketPrefix, null);
+ }
+
+ /**
+ * Returns Iterator to iterate over all buckets after prevBucket for a
+ * specific user. If prevBucket is null it returns an iterator to iterate over
+ * all the buckets of a user. The result can be restricted using bucket
+ * prefix, will return all buckets if bucket prefix is null.
+ *
+ * @param userName user name
+ * @param bucketPrefix Bucket prefix to match
+ * @param prevBucket Buckets are listed after this bucket
+ * @return {@code Iterator<OzoneBucket>}
+ */
+ public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
+ String bucketPrefix,
+ String prevBucket) {
+ return new S3BucketIterator(userName, bucketPrefix, prevBucket);
+ }
/**
* Returns Iterator to iterate over all the volumes in object store.
@@ -270,4 +303,72 @@ public class ObjectStore {
}
}
+ /**
+ * An Iterator to iterate over {@link OzoneBucket} list.
+ */
+ public class S3BucketIterator implements Iterator<OzoneBucket> {
+
+ private String bucketPrefix = null;
+ private String userName;
+
+ private Iterator<OzoneBucket> currentIterator;
+ private OzoneBucket currentValue;
+
+
+ /**
+ * Creates an Iterator to iterate over all buckets after prevBucket for
+ * a user. If prevBucket is null it returns an iterator which list all
+ * the buckets of the user.
+ * The returned buckets match bucket prefix.
+ * @param user
+ * @param bucketPrefix
+ * @param prevBucket
+ */
+ public S3BucketIterator(String user, String bucketPrefix, String
+ prevBucket) {
+ Objects.requireNonNull(user);
+ this.userName = user;
+ this.bucketPrefix = bucketPrefix;
+ this.currentValue = null;
+ this.currentIterator = getNextListOfS3Buckets(prevBucket).iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if(!currentIterator.hasNext()) {
+ currentIterator = getNextListOfS3Buckets(
+ currentValue != null ? currentValue.getName() : null)
+ .iterator();
+ }
+ return currentIterator.hasNext();
+ }
+
+ @Override
+ public OzoneBucket next() {
+ if(hasNext()) {
+ currentValue = currentIterator.next();
+ return currentValue;
+ }
+ throw new NoSuchElementException();
+ }
+
+ /**
+ * Gets the next set of bucket list using proxy.
+ * @param prevBucket
+ * @return {@code List<OzoneVolume>}
+ */
+ private List<OzoneBucket> getNextListOfS3Buckets(String prevBucket) {
+ try {
+ return proxy.listS3Buckets(userName, bucketPrefix, prevBucket,
+ listCacheSize);
+ } catch (IOException e) {
+ if (e.getMessage().contains("VOLUME_NOT_FOUND")) {
+ return new ArrayList<OzoneBucket>();
+ } else {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index a3e49e9..be5670d 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -365,6 +365,22 @@ public interface ClientProtocol {
String getOzoneBucketName(String s3BucketName) throws IOException;
/**
+ * Returns Iterator to iterate over all buckets after prevBucket for a
+ * specific user. If prevBucket is null it returns an iterator to iterate over
+ * all the buckets of a user. The result can be restricted using bucket
+ * prefix, will return all buckets if bucket prefix is null.
+ *
+ * @param userName user name
+ * @param bucketPrefix Bucket prefix to match
+ * @param prevBucket Buckets are listed after this bucket
+ * @return {@code Iterator<OzoneBucket>}
+ * @throws IOException
+ */
+ List<OzoneBucket> listS3Buckets(String userName, String bucketPrefix,
+ String prevBucket, int maxListResult)
+ throws IOException;
+
+ /**
* Close and release the resources.
*/
void close() throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index 8e106a5..8889d99 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -854,6 +854,14 @@ public class RestClient implements ClientProtocol {
"support this operation.");
}
+ @Override
+ public List<OzoneBucket> listS3Buckets(String userName, String bucketPrefix,
+ String prevBucket, int maxListResult)
+ throws IOException {
+ throw new UnsupportedOperationException("Ozone REST protocol does not " +
+ "support this operation.");
+ }
+
/**
* Adds Ozone headers to http request.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 826f04b..ea002ec 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -623,6 +623,25 @@ public class RpcClient implements ClientProtocol {
}
@Override
+ public List<OzoneBucket> listS3Buckets(String userName, String bucketPrefix,
+ String prevBucket, int maxListResult)
+ throws IOException {
+ List<OmBucketInfo> buckets = ozoneManagerClient.listS3Buckets(
+ userName, prevBucket, bucketPrefix, maxListResult);
+
+ return buckets.stream().map(bucket -> new OzoneBucket(
+ conf,
+ this,
+ bucket.getVolumeName(),
+ bucket.getBucketName(),
+ bucket.getAcls(),
+ bucket.getStorageType(),
+ bucket.getIsVersionEnabled(),
+ bucket.getCreationTime()))
+ .collect(Collectors.toList());
+ }
+
+ @Override
public void close() throws IOException {
IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient);
IOUtils.cleanupWithLogger(LOG, ozoneManagerClient);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
index a0ae455..1d4d646 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
@@ -45,7 +45,8 @@ public enum OMAction implements AuditAction {
LIST_KEYS("LIST_KEYS"),
READ_VOLUME("READ_VOLUME"),
READ_BUCKET("READ_BUCKET"),
- READ_KEY("READ_BUCKET");
+ READ_KEY("READ_BUCKET"),
+ LIST_S3BUCKETS("LIST_S3BUCKETS");
private String action;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index c021e64..e4cce65 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -281,5 +281,28 @@ public interface OzoneManagerProtocol {
*/
String getOzoneBucketMapping(String s3BucketName) throws IOException;
+ /**
+ * Returns a list of buckets represented by {@link OmBucketInfo}
+ * for the given user. Argument username is required, others
+ * are optional.
+ *
+ * @param userName
+ * user Name.
+ * @param startBucketName
+ * the start bucket name, only the buckets whose name is
+ * after this value will be included in the result.
+ * @param bucketPrefix
+ * bucket name prefix, only the buckets whose name has
+ * this prefix will be included in the result.
+ * @param maxNumOfBuckets
+ * the maximum number of buckets to return. It ensures
+ * the size of the result will not exceed this limit.
+ * @return a list of buckets.
+ * @throws IOException
+ */
+ List<OmBucketInfo> listS3Buckets(String userName, String startBucketName,
+ String bucketPrefix, int maxNumOfBuckets)
+ throws IOException;
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 94c57e5..6c8c932 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -126,6 +126,10 @@ import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.S3BucketInfoRequest;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.S3BucketInfoResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .S3ListBucketsRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .S3ListBucketsResponse;
@@ -837,6 +841,40 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
return resp.getOzoneMapping();
}
+ @Override
+ public List<OmBucketInfo> listS3Buckets(String userName, String startKey,
+ String prefix, int count)
+ throws IOException {
+ List<OmBucketInfo> buckets = new ArrayList<>();
+ S3ListBucketsRequest.Builder reqBuilder = S3ListBucketsRequest.newBuilder();
+ reqBuilder.setUserName(userName);
+ reqBuilder.setCount(count);
+ if (startKey != null) {
+ reqBuilder.setStartKey(startKey);
+ }
+ if (prefix != null) {
+ reqBuilder.setPrefix(prefix);
+ }
+ S3ListBucketsRequest request = reqBuilder.build();
+ final S3ListBucketsResponse resp;
+ try {
+ resp = rpcProxy.listS3Buckets(NULL_RPC_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+
+ if (resp.getStatus() == Status.OK) {
+ buckets.addAll(
+ resp.getBucketInfoList().stream()
+ .map(OmBucketInfo::getFromProtobuf)
+ .collect(Collectors.toList()));
+ return buckets;
+ } else {
+ throw new IOException("List S3 Buckets failed, error: "
+ + resp.getStatus());
+ }
+ }
+
/**
* Return the proxy object underlying this protocol translator.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index d3c0777..6320cf1 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -393,6 +393,18 @@ message S3DeleteBucketResponse {
required Status status = 1;
}
+message S3ListBucketsRequest {
+ required string userName = 1;
+ optional string startKey = 2;
+ optional string prefix = 3;
+ optional int32 count = 4;
+}
+
+message S3ListBucketsResponse {
+ required Status status = 1;
+ repeated BucketInfo bucketInfo = 2;
+}
+
/**
The OM service that takes care of Ozone namespace.
@@ -526,4 +538,7 @@ service OzoneManagerService {
*/
rpc getS3Bucketinfo(S3BucketInfoRequest)
returns(S3BucketInfoResponse);
+
+ rpc listS3Buckets(S3ListBucketsRequest)
+ returns(S3ListBucketsResponse);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
index 555b895..1ed5f67 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
@@ -68,6 +68,10 @@ import java.util.Iterator;
import java.util.List;
import java.util.UUID;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.either;
+import static org.junit.Assert.assertThat;
+
/**
* This class is to test all the public facing APIs of Ozone Client.
*/
@@ -222,6 +226,36 @@ public class TestOzoneRpcClient {
Assert.assertTrue(volume.getCreationTime() >= currentTime);
}
+
+ @Test
+ public void testListS3Buckets()
+ throws IOException, OzoneException {
+ String userName = "ozone100";
+ String bucketName1 = UUID.randomUUID().toString();
+ String bucketName2 = UUID.randomUUID().toString();
+ store.createS3Bucket(userName, bucketName1);
+ store.createS3Bucket(userName, bucketName2);
+ Iterator<? extends OzoneBucket> iterator = store.listS3Buckets(userName,
+ null);
+
+ while (iterator.hasNext()) {
+ assertThat(iterator.next().getName(), either(containsString(bucketName1))
+ .or(containsString(bucketName2)));
+ }
+
+ }
+
+ @Test
+ public void testListS3BucketsFail()
+ throws IOException, OzoneException {
+ String userName = "randomUser";
+ Iterator<? extends OzoneBucket> iterator = store.listS3Buckets(userName,
+ null);
+
+ Assert.assertFalse(iterator.hasNext());
+
+ }
+
@Test
public void testDeleteS3Bucket()
throws IOException, OzoneException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
index b8cfc97..1b396f9 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
@@ -60,6 +60,8 @@ public class OMMetrics {
private @Metric MutableCounterLong numKeyCommits;
private @Metric MutableCounterLong numAllocateBlockCalls;
private @Metric MutableCounterLong numGetServiceLists;
+ private @Metric MutableCounterLong numListS3Buckets;
+
// Failure Metrics
private @Metric MutableCounterLong numVolumeCreateFails;
@@ -81,6 +83,7 @@ public class OMMetrics {
private @Metric MutableCounterLong numKeyCommitFails;
private @Metric MutableCounterLong numBlockAllocateCallFails;
private @Metric MutableCounterLong numGetServiceListFails;
+ private @Metric MutableCounterLong numListS3BucketsFails;
public OMMetrics() {
}
@@ -152,6 +155,16 @@ public class OMMetrics {
numVolumeLists.incr();
}
+ public void incNumListS3Buckets() {
+ numBucketOps.incr();
+ numListS3Buckets.incr();
+ }
+
+ public void incNumListS3BucketsFails() {
+ numBucketOps.incr();
+ numListS3BucketsFails.incr();
+ }
+
public void incNumGetServiceLists() {
numGetServiceLists.incr();
}
@@ -452,6 +465,16 @@ public class OMMetrics {
return numGetServiceListFails.value();
}
+ @VisibleForTesting
+ public long getNumListS3Buckets() {
+ return numListS3Buckets.value();
+ }
+
+ @VisibleForTesting
+ public long getNumListS3BucketsFails() {
+ return numListS3BucketsFails.value();
+ }
+
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 63f0d52..da56850 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -1156,6 +1156,36 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
return s3BucketManager.getOzoneBucketMapping(s3BucketName);
}
+ @Override
+ public List<OmBucketInfo> listS3Buckets(String userName, String startKey,
+ String prefix, int maxNumOfBuckets)
+ throws IOException {
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = buildAuditMap(userName);
+ auditMap.put(OzoneConsts.START_KEY, startKey);
+ auditMap.put(OzoneConsts.PREFIX, prefix);
+ auditMap.put(OzoneConsts.MAX_NUM_OF_BUCKETS,
+ String.valueOf(maxNumOfBuckets));
+ try {
+ metrics.incNumListS3Buckets();
+ String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName);
+ return bucketManager.listBuckets(volumeName, startKey, prefix,
+ maxNumOfBuckets);
+ } catch (IOException ex) {
+ metrics.incNumListS3BucketsFails();
+ auditSuccess = false;
+ AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_S3BUCKETS,
+ auditMap, ex));
+ throw ex;
+ } finally {
+ if(auditSuccess){
+ AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction
+ .LIST_S3BUCKETS, auditMap));
+ }
+ }
+ }
+
+
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
index 4144662..f22af7f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
@@ -63,4 +63,10 @@ public interface S3BucketManager {
* @throws IOException - in case of failure to retrieve mapping.
*/
String getOzoneBucketName(String s3BucketName) throws IOException;
+
+ /**
+ * Returns volume Name for a user.
+ * @param userName
+ */
+ String getOzoneVolumeNameForUser(String userName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
index ca032c9..d6fd410 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
@@ -31,7 +31,9 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_S3_VOLUME_PREFIX;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FAILED_VOLUME_ALREADY_EXISTS;
/**
@@ -152,7 +154,7 @@ public class S3BucketManagerImpl implements S3BucketManager {
}
private String formatOzoneVolumeName(String userName) {
- return String.format("s3%s", userName);
+ return String.format(OM_S3_VOLUME_PREFIX + "%s", userName);
}
private void createOzoneVolumeIfNeeded(String userName, String volumeName)
@@ -227,4 +229,10 @@ public class S3BucketManagerImpl implements S3BucketManager {
return mapping.split("/")[1];
}
+ @Override
+ public String getOzoneVolumeNameForUser(String userName) throws IOException {
+ Objects.requireNonNull(userName, "UserName cannot be null");
+ return formatOzoneVolumeName(userName);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
index 9416056..64806ed 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
@@ -102,6 +102,10 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.S3DeleteBucketResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .S3ListBucketsResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .S3ListBucketsRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.ServiceListRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.ServiceListResponse;
@@ -627,4 +631,24 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
}
return resp.build();
}
+
+ @Override
+ public S3ListBucketsResponse listS3Buckets(RpcController controller,
+ S3ListBucketsRequest request) {
+ S3ListBucketsResponse.Builder resp = S3ListBucketsResponse.newBuilder();
+ try {
+ List<OmBucketInfo> buckets = impl.listS3Buckets(
+ request.getUserName(),
+ request.getStartKey(),
+ request.getPrefix(),
+ request.getCount());
+ for(OmBucketInfo bucket : buckets) {
+ resp.addBucketInfo(bucket.getProtobuf());
+ }
+ resp.setStatus(Status.OK);
+ } catch (IOException e) {
+ resp.setStatus(exceptionToResponseStatus(e));
+ }
+ return resp.build();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
index 75349fb..5160045 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.ozone.om;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.server.ServerUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
@@ -76,6 +78,29 @@ public class TestS3BucketManager {
}
@Test
+ public void testOzoneVolumeNameForUser() throws IOException {
+ S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
+ volumeManager, bucketManager);
+ String userName = "ozone";
+ String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName);
+ assertEquals(OzoneConsts.OM_S3_VOLUME_PREFIX + userName, volumeName);
+ }
+
+ @Test
+ public void testOzoneVolumeNameForUserFails() throws IOException {
+ S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
+ volumeManager, bucketManager);
+ String userName = null;
+ try {
+ String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName);
+ fail("testOzoneVolumeNameForUserFails failed");
+ } catch (NullPointerException ex) {
+ GenericTestUtils.assertExceptionContains("UserName cannot be null", ex);
+ }
+
+ }
+
+ @Test
public void testDeleteS3Bucket() throws IOException {
S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
volumeManager, bucketManager);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index a5af7f1..cfa2117 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.s3.endpoint;
import javax.inject.Inject;
import javax.ws.rs.NotFoundException;
import java.io.IOException;
+import java.util.Iterator;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
@@ -172,6 +173,37 @@ public class EndpointBase {
return client.getObjectStore().getOzoneBucketName(s3BucketName);
}
+ /**
+ * Returns Iterator to iterate over all buckets for a specific user.
+ * The result can be restricted using bucket prefix, will return all
+ * buckets if bucket prefix is null.
+ *
+ * @param userName
+ * @param prefix
+ * @return {@code Iterator<OzoneBucket>}
+ */
+ public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
+ String prefix) {
+ return client.getObjectStore().listS3Buckets(userName, prefix);
+ }
+
+ /**
+ * Returns Iterator to iterate over all buckets after prevBucket for a
+ * specific user. If prevBucket is null it returns an iterator to iterate
+ * over all buckets for this user. The result can be restricted using
+ * bucket prefix, will return all buckets if bucket prefix is null.
+ *
+ * @param prefix Bucket prefix to match
+ * @param previousBucket Buckets are listed after this bucket
+ * @return {@code Iterator<OzoneBucket>}
+ */
+ public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
+ String prefix,
+ String previousBucket) {
+ return client.getObjectStore().listS3Buckets(userName, prefix,
+ previousBucket);
+ }
+
public AuthenticationHeaderParser getAuthenticationHeaderParser() {
return authenticationHeaderParser;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
index 1b0e876..7ad374d 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.s3.endpoint;
import javax.ws.rs.GET;
-import javax.ws.rs.NotFoundException;
import javax.ws.rs.Path;
import java.io.IOException;
import java.time.Instant;
@@ -53,25 +52,16 @@ public class RootEndpoint extends EndpointBase {
OzoneVolume volume;
ListBucketResponse response = new ListBucketResponse();
- String volumeName = "s3" + getAuthenticationHeaderParser().getAccessKeyID();
- try {
- //TODO: we need a specific s3bucketlist endpoint instead
- // of reimplement the naming convention here
- volume = getVolume(volumeName);
- } catch (NotFoundException ex) {
- return response;
- } catch (IOException e) {
- throw e;
- }
-
- Iterator<? extends OzoneBucket> volABucketIter = volume.listBuckets(null);
+ String userName = getAuthenticationHeaderParser().getAccessKeyID();
+ Iterator<? extends OzoneBucket> bucketIterator = listS3Buckets(userName,
+ null);
- while (volABucketIter.hasNext()) {
- OzoneBucket next = volABucketIter.next();
+ while (bucketIterator.hasNext()) {
+ OzoneBucket next = bucketIterator.next();
BucketMetadata bucketMetadata = new BucketMetadata();
bucketMetadata.setName(next.getName());
- bucketMetadata.setCreationDate(
- Instant.ofEpochMilli(next.getCreationTime()));
+ bucketMetadata.setCreationDate(Instant.ofEpochMilli(next
+ .getCreationTime()));
response.addBucket(bucketMetadata);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java
index 4984d58..de15a40 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java
@@ -20,10 +20,7 @@
package org.apache.hadoop.ozone.client;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
+import java.util.*;
import java.util.stream.Collectors;
/**
@@ -38,6 +35,7 @@ public class ObjectStoreStub extends ObjectStore {
private Map<String, OzoneVolumeStub> volumes = new HashMap<>();
private Map<String, String> bucketVolumeMap = new HashMap<>();
private Map<String, Boolean> bucketEmptyStatus = new HashMap<>();
+ private Map<String, List<OzoneBucket>> userBuckets = new HashMap<>();
@Override
public void createVolume(String volumeName) throws IOException {
@@ -113,8 +111,8 @@ public class ObjectStoreStub extends ObjectStore {
@Override
public void createS3Bucket(String userName, String s3BucketName) throws
IOException {
+ String volumeName = "s3" + userName;
if (bucketVolumeMap.get(s3BucketName) == null) {
- String volumeName = "s3"+userName;
bucketVolumeMap.put(s3BucketName, volumeName + "/" + s3BucketName);
bucketEmptyStatus.put(s3BucketName, true);
createVolume(volumeName);
@@ -122,6 +120,70 @@ public class ObjectStoreStub extends ObjectStore {
} else {
throw new IOException("BUCKET_ALREADY_EXISTS");
}
+
+ if (userBuckets.get(userName) == null) {
+ List<OzoneBucket> ozoneBuckets = new ArrayList<>();
+ ozoneBuckets.add(volumes.get(volumeName).getBucket(s3BucketName));
+ userBuckets.put(userName, ozoneBuckets);
+ } else {
+ userBuckets.get(userName).add(volumes.get(volumeName).getBucket(
+ s3BucketName));
+ }
+ }
+
+ public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
+ String bucketPrefix) {
+ if (userBuckets.get(userName) == null) {
+ return new ArrayList<OzoneBucket>().iterator();
+ } else {
+ return userBuckets.get(userName).parallelStream()
+ .filter(ozoneBucket -> {
+ if (bucketPrefix != null) {
+ return ozoneBucket.getName().startsWith(bucketPrefix);
+ } else {
+ return true;
+ }
+ }).collect(Collectors.toList())
+ .iterator();
+ }
+ }
+
+ public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
+ String bucketPrefix,
+ String prevBucket) {
+
+ if (userBuckets.get(userName) == null) {
+ return new ArrayList<OzoneBucket>().iterator();
+ } else {
+ //Sort buckets lexicographically
+ userBuckets.get(userName).sort(
+ (bucket1, bucket2) -> {
+ int compare = bucket1.getName().compareTo(bucket2.getName());
+ if (compare < 0) {
+ return -1;
+ } else if (compare == 0) {
+ return 0;
+ } else {
+ return 1;
+ }
+ });
+ return userBuckets.get(userName).stream()
+ .filter(ozoneBucket -> {
+ if (prevBucket != null) {
+ return ozoneBucket.getName().compareTo(prevBucket) > 0;
+ } else {
+ return true;
+ }
+ })
+ .filter(ozoneBucket -> {
+ if (bucketPrefix != null) {
+ return ozoneBucket.getName().startsWith(bucketPrefix);
+ } else {
+ return true;
+ }
+ }).collect(Collectors.toList())
+ .iterator();
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
index bd54896..2d0504d 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
@@ -51,7 +51,6 @@ public class TestObjectGet {
client.getObjectStore().createS3Bucket("bilbo", "b1");
String volumeName = client.getObjectStore().getOzoneVolumeName("b1");
OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
- volume.createBucket("b1");
OzoneBucket bucket =
volume.getBucket("b1");
OzoneOutputStream keyStream =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
index 80cad3a..4f76067 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
@@ -20,15 +20,12 @@
package org.apache.hadoop.ozone.s3.endpoint;
-
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClientStub;
-import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.s3.header.AuthenticationHeaderParser;
import org.apache.commons.lang3.RandomStringUtils;
import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.ozone.s3.header.AuthenticationHeaderParser;
import org.junit.Before;
import org.junit.Test;
@@ -39,7 +36,6 @@ public class TestRootList {
private OzoneClientStub clientStub;
private ObjectStore objectStoreStub;
- private OzoneVolume volumeStub;
private RootEndpoint rootEndpoint;
private String userName = "ozone";
@@ -49,9 +45,6 @@ public class TestRootList {
//Create client stub and object store stub.
clientStub = new OzoneClientStub();
objectStoreStub = clientStub.getObjectStore();
- String volumeName = "s3" + userName;
- objectStoreStub.createVolume(volumeName);
- volumeStub = objectStoreStub.getVolume(volumeName);
// Create HeadBucket and setClient to OzoneClientStub
rootEndpoint = new RootEndpoint();
@@ -71,7 +64,7 @@ public class TestRootList {
String bucketBaseName = "bucket-";
for(int i = 0; i < 10; i++) {
- volumeStub.createBucket(
+ objectStoreStub.createS3Bucket(userName,
bucketBaseName + RandomStringUtils.randomNumeric(3));
}
response = rootEndpoint.get();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org