You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2017/10/02 18:52:15 UTC
hadoop git commit: HDFS-12060. Ozone: OzoneClient: Add list calls in
RpcClient. Contributed by Nandakumar.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 3bf591fd8 -> 579434a83
HDFS-12060. Ozone: OzoneClient: Add list calls in RpcClient. Contributed by Nandakumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/579434a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/579434a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/579434a8
Branch: refs/heads/HDFS-7240
Commit: 579434a83fbd7ee250182f4ccb72a5bd99ab309f
Parents: 3bf591f
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Mon Oct 2 11:50:59 2017 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Mon Oct 2 11:50:59 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/ozone/OzoneConfigKeys.java | 7 +
.../apache/hadoop/ozone/client/ObjectStore.java | 122 ++++++++-
.../apache/hadoop/ozone/client/OzoneBucket.java | 95 ++++++-
.../apache/hadoop/ozone/client/OzoneClient.java | 8 +-
.../hadoop/ozone/client/OzoneClientFactory.java | 3 +-
.../hadoop/ozone/client/OzoneClientUtils.java | 17 +-
.../apache/hadoop/ozone/client/OzoneVolume.java | 99 +++++++-
.../ozone/client/protocol/ClientProtocol.java | 54 ++--
.../hadoop/ozone/client/rest/RestClient.java | 17 +-
.../hadoop/ozone/client/rpc/RpcClient.java | 97 ++++++--
.../ozone/client/rpc/TestOzoneRpcClient.java | 246 ++++++++++++++++++-
11 files changed, 686 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579434a8/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index d51c533..67161e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -121,6 +121,13 @@ public final class OzoneConfigKeys {
public static final int OZONE_CLIENT_CONNECTION_TIMEOUT_MS_DEFAULT = 5000;
/**
+ * Configuration property to configure the cache size of client list calls.
+ */
+ public static final String OZONE_CLIENT_LIST_CACHE_SIZE =
+ "ozone.client.list.cache";
+ public static final int OZONE_CLIENT_LIST_CACHE_SIZE_DEFAULT = 1000;
+
+ /**
* Configuration properties for Ozone Block Deleting Service.
*/
public static final String OZONE_BLOCK_DELETING_SERVICE_INTERVAL_MS =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579434a8/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index 49addf7..2e661db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -19,9 +19,15 @@
package org.apache.hadoop.ozone.client;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.security.UserGroupInformation;
import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
/**
* ObjectStore class is responsible for the client operations that can be
@@ -36,11 +42,18 @@ public class ObjectStore {
private final ClientProtocol proxy;
/**
- * Creates an instance of ObjectStore with the proxy.
- * @param proxy ClientProtocol proxy
+ * Cache size to be used for listVolume calls.
*/
- public ObjectStore(ClientProtocol proxy) {
+ private int listCacheSize;
+
+ /**
+ * Creates an instance of ObjectStore.
+ * @param conf Configuration object.
+ * @param proxy ClientProtocol proxy.
+ */
+ public ObjectStore(Configuration conf, ClientProtocol proxy) {
this.proxy = proxy;
+ this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
}
/**
@@ -78,10 +91,41 @@ public class ObjectStore {
Preconditions.checkNotNull(volumeName);
OzoneClientUtils.verifyResourceName(volumeName);
OzoneVolume volume = proxy.getVolumeDetails(volumeName);
- volume.setClientProxy(proxy);
return volume;
}
+
+ /**
+ * Returns Iterator to iterate over all the volumes in object store.
+ * The result can be restricted using volume prefix, will return all
+ * volumes if volume prefix is null.
+ *
+ * @param volumePrefix Volume prefix to match
+ * @return {@code Iterator<OzoneVolume>}
+ */
+ public Iterator<OzoneVolume> listVolumes(String volumePrefix)
+ throws IOException {
+ return new VolumeIterator(volumePrefix);
+ }
+
+ /**
+ * Returns Iterator to iterate over the List of volumes owned by a specific
+ * user. The result can be restricted using volume prefix, will return all
+ * volumes if volume prefix is null. If user is null, returns the volume of
+ * current user.
+ *
+ * @param user User Name
+ * @param volumePrefix Volume prefix to match
+ * @return {@code Iterator<OzoneVolume>}
+ */
+ public Iterator<OzoneVolume> listVolumes(String user, String volumePrefix)
+ throws IOException {
+ if(Strings.isNullOrEmpty(user)) {
+ user = UserGroupInformation.getCurrentUser().getShortUserName();
+ }
+ return new VolumeIterator(user, volumePrefix);
+ }
+
/**
* Deletes the volume.
* @param volumeName Name of the volume.
@@ -92,4 +136,74 @@ public class ObjectStore {
OzoneClientUtils.verifyResourceName(volumeName);
proxy.deleteVolume(volumeName);
}
+
+ /**
+ * An Iterator to iterate over {@link OzoneVolume} list.
+ */
+ private class VolumeIterator implements Iterator<OzoneVolume> {
+
+ private String user = null;
+ private String volPrefix = null;
+
+ private Iterator<OzoneVolume> currentIterator;
+ private OzoneVolume currentValue;
+
+ /**
+ * Creates an Iterator to iterate over all volumes in the cluster,
+ * which matches the volume prefix.
+ * @param volPrefix prefix to match
+ */
+ VolumeIterator(String volPrefix) {
+ this(null, volPrefix);
+ }
+
+ /**
+ * Creates an Iterator to iterate over all volumes of the user,
+ * which matches volume prefix.
+ * @param user user name
+ * @param volPrefix volume prefix to match
+ */
+ VolumeIterator(String user, String volPrefix) {
+ this.user = user;
+ this.volPrefix = volPrefix;
+ this.currentValue = null;
+ this.currentIterator = getNextListOfVolumes(null).iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if(!currentIterator.hasNext()) {
+ currentIterator = getNextListOfVolumes(
+ currentValue.getName()).iterator();
+ }
+ return currentIterator.hasNext();
+ }
+
+ @Override
+ public OzoneVolume next() {
+ if(hasNext()) {
+ currentValue = currentIterator.next();
+ return currentValue;
+ }
+ throw new NoSuchElementException();
+ }
+
+ /**
+ * Returns the next set of volume list using proxy.
+ * @param prevVolume previous volume, this will be excluded from the result
+ * @return {@code List<OzoneVolume>}
+ */
+ private List<OzoneVolume> getNextListOfVolumes(String prevVolume) {
+ try {
+ //if user is null, we do list of all volumes.
+ if(user != null) {
+ return proxy.listVolumes(user, volPrefix, prevVolume, listCacheSize);
+ }
+ return proxy.listVolumes(volPrefix, prevVolume, listCacheSize);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579434a8/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index 6d53c5b..5c0a3f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.client;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
@@ -27,7 +28,9 @@ import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.OzoneAcl;
import java.io.IOException;
+import java.util.Iterator;
import java.util.List;
+import java.util.NoSuchElementException;
/**
* A class that encapsulates OzoneBucket.
@@ -35,6 +38,11 @@ import java.util.List;
public class OzoneBucket {
/**
+ * The proxy used for connecting to the cluster and perform
+ * client operations.
+ */
+ private final ClientProtocol proxy;
+ /**
* Name of the volume in which the bucket belongs to.
*/
private final String volumeName;
@@ -59,35 +67,31 @@ public class OzoneBucket {
private Boolean versioning;
/**
- * The proxy used for connecting to the cluster and perform
- * client operations.
+ * Cache size to be used for listKey calls.
*/
- private ClientProtocol proxy;
+ private int listCacheSize;
/**
* Constructs OzoneBucket instance.
+ * @param conf Configuration object.
+ * @param proxy ClientProtocol proxy.
* @param volumeName Name of the volume the bucket belongs to.
* @param bucketName Name of the bucket.
* @param acls ACLs associated with the bucket.
* @param storageType StorageType of the bucket.
* @param versioning versioning status of the bucket.
*/
- public OzoneBucket(String volumeName, String bucketName,
+ public OzoneBucket(Configuration conf, ClientProtocol proxy,
+ String volumeName, String bucketName,
List<OzoneAcl> acls, StorageType storageType,
Boolean versioning) {
+ this.proxy = proxy;
this.volumeName = volumeName;
this.name = bucketName;
this.acls = acls;
this.storageType = storageType;
this.versioning = versioning;
- }
-
- /**
- * Sets the proxy using which client operations are performed.
- * @param clientProxy
- */
- public void setClientProxy(ClientProtocol clientProxy) {
- this.proxy = clientProxy;
+ this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
}
/**
@@ -222,6 +226,18 @@ public class OzoneBucket {
}
/**
+ * Returns Iterator to iterate over all keys in the bucket.
+ * The result can be restricted using key prefix, will return all
+ * keys if key prefix is null.
+ *
+ * @param keyPrefix Bucket prefix to match
+ * @return {@code Iterator<OzoneKey>}
+ */
+ public Iterator<OzoneKey> listKeys(String keyPrefix) {
+ return new KeyIterator(keyPrefix);
+ }
+
+ /**
* Deletes key from the bucket.
* @param key Name of the key to be deleted.
* @throws IOException
@@ -231,4 +247,59 @@ public class OzoneBucket {
Preconditions.checkNotNull(key);
proxy.deleteKey(volumeName, name, key);
}
+
+ /**
+ * An Iterator to iterate over {@link OzoneKey} list.
+ */
+ private class KeyIterator implements Iterator<OzoneKey> {
+
+ private String keyPrefix = null;
+
+ private Iterator<OzoneKey> currentIterator;
+ private OzoneKey currentValue;
+
+
+ /**
+ * Creates an Iterator to iterate over all keys in the bucket,
+ * which matches volume prefix.
+ * @param keyPrefix
+ */
+ KeyIterator(String keyPrefix) {
+ this.keyPrefix = keyPrefix;
+ this.currentValue = null;
+ this.currentIterator = getNextListOfKeys(null).iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if(!currentIterator.hasNext()) {
+ currentIterator = getNextListOfKeys(
+ currentValue.getName()).iterator();
+ }
+ return currentIterator.hasNext();
+ }
+
+ @Override
+ public OzoneKey next() {
+ if(hasNext()) {
+ currentValue = currentIterator.next();
+ return currentValue;
+ }
+ throw new NoSuchElementException();
+ }
+
+ /**
+ * Gets the next set of key list using proxy.
+ * @param prevKey
+ * @return {@code List<OzoneVolume>}
+ */
+ private List<OzoneKey> getNextListOfKeys(String prevKey) {
+ try {
+ return proxy.listKeys(volumeName, name, keyPrefix, prevKey,
+ listCacheSize);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579434a8/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java
index b7c3a11..f191507 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.client;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import java.io.Closeable;
@@ -75,11 +76,12 @@ public class OzoneClient implements Closeable {
/**
* Creates a new OzoneClient object, generally constructed
* using {@link OzoneClientFactory}.
- * @param proxy
+ * @param conf Configuration object
+ * @param proxy ClientProtocol proxy instance
*/
- public OzoneClient(ClientProtocol proxy) {
+ public OzoneClient(Configuration conf, ClientProtocol proxy) {
this.proxy = proxy;
- this.objectStore = new ObjectStore(this.proxy);
+ this.objectStore = new ObjectStore(conf, this.proxy);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579434a8/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
index cda95a5..75c99c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.Proxy;
@@ -107,7 +106,7 @@ public final class OzoneClientFactory {
ClientProtocol proxy = (ClientProtocol) Proxy.newProxyInstance(
OzoneClientInvocationHandler.class.getClassLoader(),
new Class<?>[]{ClientProtocol.class}, clientHandler);
- return new OzoneClient(proxy);
+ return new OzoneClient(configuration, proxy);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579434a8/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
index 19bfc18..f0465f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
@@ -54,11 +54,12 @@ import static org.apache.hadoop.cblock.CBlockConfigKeys
.DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys
.DFS_CBLOCK_JSCSI_PORT_DEFAULT;
-
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
+ .OZONE_KSM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
.OZONE_KSM_BIND_HOST_DEFAULT;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_PORT_DEFAULT;
+import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
+ .OZONE_KSM_PORT_DEFAULT;
import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_DEADNODE_INTERVAL_DEFAULT;
import static org.apache.hadoop.scm.ScmConfigKeys
@@ -440,6 +441,16 @@ public final class OzoneClientUtils {
}
/**
+ * Returns the cache value to be used for list calls.
+ * @param conf Configuration object
+ * @return list cache size
+ */
+ public static int getListCacheSize(Configuration conf) {
+ return conf.getInt(OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE_DEFAULT);
+ }
+
+ /**
* Retrieve the port number, trying the supplied config keys in order.
* Each config value may be absent, or if present in the format
* host:port (the :port part is optional).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579434a8/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
index 64c35bc..f074fb2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
@@ -19,11 +19,14 @@
package org.apache.hadoop.ozone.client;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import java.io.IOException;
+import java.util.Iterator;
import java.util.List;
+import java.util.NoSuchElementException;
/**
* A class that encapsulates OzoneVolume.
@@ -31,6 +34,12 @@ import java.util.List;
public class OzoneVolume {
/**
+ * The proxy used for connecting to the cluster and perform
+ * client operations.
+ */
+ private final ClientProtocol proxy;
+
+ /**
* Name of the Volume.
*/
private final String name;
@@ -52,22 +61,28 @@ public class OzoneVolume {
*/
private List<OzoneAcl> acls;
- private ClientProtocol proxy;
+ private int listCacheSize;
/**
- * Constructs OzoneVolume.
+ * Constructs OzoneVolume instance.
+ * @param conf Configuration object.
+ * @param proxy ClientProtocol proxy.
+ * @param name Name of the volume.
+ * @param admin Volume admin.
+ * @param owner Volume owner.
+ * @param quotaInBytes Volume quota in bytes.
+ * @param acls ACLs associated with the volume.
*/
- public OzoneVolume(String name, String admin, String owner,
- long quotaInBytes, List<OzoneAcl> acls) {
+ public OzoneVolume(Configuration conf, ClientProtocol proxy, String name,
+ String admin, String owner, long quotaInBytes,
+ List<OzoneAcl> acls) {
+ this.proxy = proxy;
this.name = name;
this.admin = admin;
this.owner = owner;
this.quotaInBytes = quotaInBytes;
this.acls = acls;
- }
-
- public void setClientProxy(ClientProtocol clientProxy) {
- this.proxy = clientProxy;
+ this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
}
/**
@@ -178,11 +193,22 @@ public class OzoneVolume {
Preconditions.checkNotNull(bucketName);
OzoneClientUtils.verifyResourceName(bucketName);
OzoneBucket bucket = proxy.getBucketDetails(name, bucketName);
- bucket.setClientProxy(proxy);
return bucket;
}
/**
+ * Returns Iterator to iterate over all buckets in the volume.
+ * The result can be restricted using bucket prefix, will return all
+ * buckets if bucket prefix is null.
+ *
+ * @param bucketPrefix Bucket prefix to match
+ * @return {@code Iterator<OzoneBucket>}
+ */
+ public Iterator<OzoneBucket> listBuckets(String bucketPrefix) {
+ return new BucketIterator(bucketPrefix);
+ }
+
+ /**
* Deletes the Bucket from this Volume.
* @param bucketName Name of the Bucket
* @throws IOException
@@ -193,4 +219,59 @@ public class OzoneVolume {
OzoneClientUtils.verifyResourceName(bucketName);
proxy.deleteBucket(name, bucketName);
}
+
+
+ /**
+ * An Iterator to iterate over {@link OzoneBucket} list.
+ */
+ private class BucketIterator implements Iterator<OzoneBucket> {
+
+ private String bucketPrefix = null;
+
+ private Iterator<OzoneBucket> currentIterator;
+ private OzoneBucket currentValue;
+
+
+ /**
+ * Creates an Iterator to iterate over all buckets in the volume,
+ * which matches volume prefix.
+ * @param bucketPrefix
+ */
+ BucketIterator(String bucketPrefix) {
+ this.bucketPrefix = bucketPrefix;
+ this.currentValue = null;
+ this.currentIterator = getNextListOfBuckets(null).iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if(!currentIterator.hasNext()) {
+ currentIterator = getNextListOfBuckets(
+ currentValue.getName()).iterator();
+ }
+ return currentIterator.hasNext();
+ }
+
+ @Override
+ public OzoneBucket next() {
+ if(hasNext()) {
+ currentValue = currentIterator.next();
+ return currentValue;
+ }
+ throw new NoSuchElementException();
+ }
+
+ /**
+ * Gets the next set of bucket list using proxy.
+ * @param prevBucket
+ * @return {@code List<OzoneVolume>}
+ */
+ private List<OzoneBucket> getNextListOfBuckets(String prevBucket) {
+ try {
+ return proxy.listBuckets(name, bucketPrefix, prevBucket, listCacheSize);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579434a8/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 54aa3f9..2bb1ec2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import java.io.IOException;
-import java.util.Iterator;
import java.util.List;
/**
@@ -106,22 +105,37 @@ public interface ClientProtocol {
void deleteVolume(String volumeName) throws IOException;
/**
- * Returns the List of Volumes owned by current user.
+ * Lists all volumes in the cluster that matches the volumePrefix,
+ * size of the returned list depends on maxListResult. If volume prefix
+ * is null, returns all the volumes. The caller has to make multiple calls
+ * to read all volumes.
+ *
* @param volumePrefix Volume prefix to match
- * @return {@link OzoneVolume} Iterator
+ * @param prevVolume Starting point of the list, this volume is excluded
+ * @param maxListResult Max number of volumes to return.
+ * @return {@code List<OzoneVolume>}
* @throws IOException
*/
- Iterator<OzoneVolume> listVolumes(String volumePrefix)
+ List<OzoneVolume> listVolumes(String volumePrefix, String prevVolume,
+ int maxListResult)
throws IOException;
/**
- * Returns the List of Volumes owned by the specific user.
- * @param volumePrefix Volume prefix to match
+ * Lists all volumes in the cluster that are owned by the specified
+ * user and matches the volumePrefix, size of the returned list depends on
+ * maxListResult. If the user is null, return volumes owned by current user.
+ * If volume prefix is null, returns all the volumes. The caller has to make
+ * multiple calls to read all volumes.
+ *
* @param user User Name
- * @return {@link OzoneVolume} Iterator
+ * @param volumePrefix Volume prefix to match
+ * @param prevVolume Starting point of the list, this volume is excluded
+ * @param maxListResult Max number of volumes to return.
+ * @return {@code List<OzoneVolume>}
* @throws IOException
*/
- Iterator<OzoneVolume> listVolumes(String volumePrefix, String user)
+ List<OzoneVolume> listVolumes(String user, String volumePrefix,
+ String prevVolume, int maxListResult)
throws IOException;
/**
@@ -219,13 +233,18 @@ public interface ClientProtocol {
throws IOException;
/**
- * Returns the List of Buckets in the Volume.
+ * Returns the List of Buckets in the Volume that matches the bucketPrefix,
+ * size of the returned list depends on maxListResult. The caller has to make
+ * multiple calls to read all volumes.
* @param volumeName Name of the Volume
* @param bucketPrefix Bucket prefix to match
- * @return {@link OzoneBucket} Iterator
+ * @param prevBucket Starting point of the list, this bucket is excluded
+ * @param maxListResult Max number of buckets to return.
+ * @return {@code List<OzoneBucket>}
* @throws IOException
*/
- Iterator<OzoneBucket> listBuckets(String volumeName, String bucketPrefix)
+ List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
+ String prevBucket, int maxListResult)
throws IOException;
/**
@@ -265,14 +284,19 @@ public interface ClientProtocol {
/**
- * Returns list of {@link OzoneKey} in {Volume/Bucket}.
+ * Returns list of Keys in {Volume/Bucket} that matches the keyPrefix,
+ * size of the returned list depends on maxListResult. The caller has
+ * to make multiple calls to read all keys.
* @param volumeName Name of the Volume
* @param bucketName Name of the Bucket
- * @return {@link OzoneKey} Iterator
+ * @param keyPrefix Bucket prefix to match
+ * @param prevKey Starting point of the list, this key is excluded
+ * @param maxListResult Max number of buckets to return.
+ * @return {@code List<OzoneKey>}
* @throws IOException
*/
- Iterator<OzoneKey> listKeys(String volumeName, String bucketName,
- String keyPrefix)
+ List<OzoneKey> listKeys(String volumeName, String bucketName,
+ String keyPrefix, String prevKey, int maxListResult)
throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579434a8/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index b525aa8..b28b06a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import java.io.IOException;
-import java.util.Iterator;
import java.util.List;
/**
@@ -93,13 +92,15 @@ public class RestClient implements ClientProtocol {
}
@Override
- public Iterator<OzoneVolume> listVolumes(String volumePrefix)
+ public List<OzoneVolume> listVolumes(String volumePrefix, String prevKey,
+ int maxListResult)
throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
}
@Override
- public Iterator<OzoneVolume> listVolumes(String volumePrefix, String user)
+ public List<OzoneVolume> listVolumes(String user, String volumePrefix,
+ String prevKey, int maxListResult)
throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
}
@@ -164,8 +165,9 @@ public class RestClient implements ClientProtocol {
}
@Override
- public Iterator<OzoneBucket> listBuckets(
- String volumeName, String bucketPrefix) throws IOException {
+ public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
+ String prevBucket, int maxListResult)
+ throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
}
@@ -190,8 +192,9 @@ public class RestClient implements ClientProtocol {
}
@Override
- public Iterator<OzoneKey> listKeys(
- String volumeName, String bucketName, String keyPrefix)
+ public List<OzoneKey> listKeys(String volumeName, String bucketName,
+ String keyPrefix, String prevKey,
+ int maxListResult)
throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579434a8/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 68e9f5a..8a85d37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -66,7 +66,6 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
@@ -81,14 +80,13 @@ public class RpcClient implements ClientProtocol {
private static final Logger LOG =
LoggerFactory.getLogger(RpcClient.class);
+ private final Configuration conf;
private final StorageContainerLocationProtocolClientSideTranslatorPB
storageContainerLocationClient;
private final KeySpaceManagerProtocolClientSideTranslatorPB
keySpaceManagerClient;
private final XceiverClientManager xceiverClientManager;
private final int chunkSize;
-
-
private final UserGroupInformation ugi;
private final OzoneAcl.OzoneACLRights userRights;
private final OzoneAcl.OzoneACLRights groupRights;
@@ -100,12 +98,12 @@ public class RpcClient implements ClientProtocol {
*/
public RpcClient(Configuration conf) throws IOException {
Preconditions.checkNotNull(conf);
+ this.conf = conf;
this.ugi = UserGroupInformation.getCurrentUser();
this.userRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_USER_RIGHTS,
KSMConfigKeys.OZONE_KSM_USER_RIGHTS_DEFAULT);
this.groupRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS,
KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS_DEFAULT);
-
long ksmVersion =
RPC.getProtocolVersion(KeySpaceManagerProtocolPB.class);
InetSocketAddress ksmAddress = OzoneClientUtils
@@ -219,10 +217,15 @@ public class RpcClient implements ClientProtocol {
throws IOException {
Preconditions.checkNotNull(volumeName);
KsmVolumeArgs volume = keySpaceManagerClient.getVolumeInfo(volumeName);
- return new OzoneVolume(volume.getVolume(), volume.getAdminName(),
- volume.getOwnerName(), volume.getQuotaInBytes(),
+ return new OzoneVolume(
+ conf,
+ this,
+ volume.getVolume(),
+ volume.getAdminName(),
+ volume.getOwnerName(),
+ volume.getQuotaInBytes(),
volume.getAclMap().ozoneAclGetProtobuf().stream().
- map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
+ map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
}
@Override
@@ -238,15 +241,41 @@ public class RpcClient implements ClientProtocol {
}
@Override
- public Iterator<OzoneVolume> listVolumes(String volumePrefix)
+ public List<OzoneVolume> listVolumes(String volumePrefix, String prevVolume,
+ int maxListResult)
throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
+ List<KsmVolumeArgs> volumes = keySpaceManagerClient.listAllVolumes(
+ volumePrefix, prevVolume, maxListResult);
+
+ return volumes.stream().map(volume -> new OzoneVolume(
+ conf,
+ this,
+ volume.getVolume(),
+ volume.getAdminName(),
+ volume.getOwnerName(),
+ volume.getQuotaInBytes(),
+ volume.getAclMap().ozoneAclGetProtobuf().stream().
+ map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
+ .collect(Collectors.toList());
}
@Override
- public Iterator<OzoneVolume> listVolumes(String volumePrefix, String user)
+ public List<OzoneVolume> listVolumes(String user, String volumePrefix,
+ String prevVolume, int maxListResult)
throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
+ List<KsmVolumeArgs> volumes = keySpaceManagerClient.listVolumeByUser(
+ user, volumePrefix, prevVolume, maxListResult);
+
+ return volumes.stream().map(volume -> new OzoneVolume(
+ conf,
+ this,
+ volume.getVolume(),
+ volume.getAdminName(),
+ volume.getOwnerName(),
+ volume.getQuotaInBytes(),
+ volume.getAclMap().ozoneAclGetProtobuf().stream().
+ map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
+ .collect(Collectors.toList());
}
@Override
@@ -371,17 +400,32 @@ public class RpcClient implements ClientProtocol {
Preconditions.checkNotNull(bucketName);
KsmBucketInfo bucketArgs =
keySpaceManagerClient.getBucketInfo(volumeName, bucketName);
- return new OzoneBucket(bucketArgs.getVolumeName(),
- bucketArgs.getBucketName(),
- bucketArgs.getAcls(),
- bucketArgs.getStorageType(),
- bucketArgs.getIsVersionEnabled());
+ return new OzoneBucket(
+ conf,
+ this,
+ bucketArgs.getVolumeName(),
+ bucketArgs.getBucketName(),
+ bucketArgs.getAcls(),
+ bucketArgs.getStorageType(),
+ bucketArgs.getIsVersionEnabled());
}
@Override
- public Iterator<OzoneBucket> listBuckets(
- String volumeName, String bucketPrefix) throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
+ public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
+ String prevBucket, int maxListResult)
+ throws IOException {
+ List<KsmBucketInfo> buckets = keySpaceManagerClient.listBuckets(
+ volumeName, prevBucket, bucketPrefix, maxListResult);
+
+ return buckets.stream().map(bucket -> new OzoneBucket(
+ conf,
+ this,
+ bucket.getVolumeName(),
+ bucket.getBucketName(),
+ bucket.getAcls(),
+ bucket.getStorageType(),
+ bucket.getIsVersionEnabled()))
+ .collect(Collectors.toList());
}
@Override
@@ -441,10 +485,19 @@ public class RpcClient implements ClientProtocol {
}
@Override
- public Iterator<OzoneKey> listKeys(
- String volumeName, String bucketName, String keyPrefix)
+ public List<OzoneKey> listKeys(String volumeName, String bucketName,
+ String keyPrefix, String prevKey,
+ int maxListResult)
throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
+ List<KsmKeyInfo> keys = keySpaceManagerClient.listKeys(
+ volumeName, bucketName, prevKey, keyPrefix, maxListResult);
+
+ return keys.stream().map(key -> new OzoneKey(
+ key.getVolumeName(),
+ key.getBucketName(),
+ key.getKeyName(),
+ key.getDataSize()))
+ .collect(Collectors.toList());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579434a8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
index b07e72c..b8b42e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.client.rpc;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneAcl;
@@ -45,6 +46,7 @@ import org.junit.rules.ExpectedException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import java.util.UUID;
@@ -61,7 +63,7 @@ public class TestOzoneRpcClient {
private static ObjectStore store = null;
/**
- * Create a MiniDFSCluster for testing.
+ * Create a MiniOzoneCluster for testing.
* <p>
* Ozone is made active by setting OZONE_ENABLED = true and
* OZONE_HANDLER_TYPE_KEY = "distributed"
@@ -403,8 +405,248 @@ public class TestOzoneRpcClient {
bucket.getKey(keyName);
}
+ @Test
+ public void listVolumeTest() throws IOException, OzoneException {
+ String volBase = "vol-" + RandomStringUtils.randomNumeric(3);
+ //Create 10 volume vol-<random>-a-0-<random> to vol-<random>-a-9-<random>
+ String volBaseNameA = volBase + "-a-";
+ for(int i = 0; i < 10; i++) {
+ store.createVolume(
+ volBaseNameA + i + "-" + RandomStringUtils.randomNumeric(5));
+ }
+ //Create 10 volume vol-<random>-b-0-<random> to vol-<random>-b-9-<random>
+ String volBaseNameB = volBase + "-b-";
+ for(int i = 0; i < 10; i++) {
+ store.createVolume(
+ volBaseNameB + i + "-" + RandomStringUtils.randomNumeric(5));
+ }
+ Iterator<OzoneVolume> volIterator = store.listVolumes(volBase);
+ int totalVolumeCount = 0;
+ while(volIterator.hasNext()) {
+ volIterator.next();
+ totalVolumeCount++;
+ }
+ Assert.assertEquals(20, totalVolumeCount);
+ Iterator<OzoneVolume> volAIterator = store.listVolumes(volBaseNameA);
+ for(int i = 0; i < 10; i++) {
+ Assert.assertTrue(volAIterator.next().getName()
+ .startsWith(volBaseNameA + i + "-"));
+ }
+ Assert.assertFalse(volAIterator.hasNext());
+ Iterator<OzoneVolume> volBIterator = store.listVolumes(volBaseNameB);
+ for(int i = 0; i < 10; i++) {
+ Assert.assertTrue(volBIterator.next().getName()
+ .startsWith(volBaseNameB + i + "-"));
+ }
+ Assert.assertFalse(volBIterator.hasNext());
+ Iterator<OzoneVolume> iter = store.listVolumes(volBaseNameA + "1-");
+ Assert.assertTrue(iter.next().getName().startsWith(volBaseNameA + "1-"));
+ Assert.assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void listBucketTest()
+ throws IOException, OzoneException {
+ String volumeA = "vol-a-" + RandomStringUtils.randomNumeric(5);
+ String volumeB = "vol-b-" + RandomStringUtils.randomNumeric(5);
+ store.createVolume(volumeA);
+ store.createVolume(volumeB);
+ OzoneVolume volA = store.getVolume(volumeA);
+ OzoneVolume volB = store.getVolume(volumeB);
+
+ //Create 10 buckets in vol-a-<random> and 10 in vol-b-<random>
+ String bucketBaseNameA = "bucket-a-";
+ for(int i = 0; i < 10; i++) {
+ volA.createBucket(
+ bucketBaseNameA + i + "-" + RandomStringUtils.randomNumeric(5));
+ volB.createBucket(
+ bucketBaseNameA + i + "-" + RandomStringUtils.randomNumeric(5));
+ }
+ //Create 10 buckets in vol-a-<random> and 10 in vol-b-<random>
+ String bucketBaseNameB = "bucket-b-";
+ for(int i = 0; i < 10; i++) {
+ volA.createBucket(
+ bucketBaseNameB + i + "-" + RandomStringUtils.randomNumeric(5));
+ volB.createBucket(
+ bucketBaseNameB + i + "-" + RandomStringUtils.randomNumeric(5));
+ }
+ Iterator<OzoneBucket> volABucketIter =
+ volA.listBuckets("bucket-");
+ int volABucketCount = 0;
+ while(volABucketIter.hasNext()) {
+ volABucketIter.next();
+ volABucketCount++;
+ }
+ Assert.assertEquals(20, volABucketCount);
+ Iterator<OzoneBucket> volBBucketIter =
+ volA.listBuckets("bucket-");
+ int volBBucketCount = 0;
+ while(volBBucketIter.hasNext()) {
+ volBBucketIter.next();
+ volBBucketCount++;
+ }
+ Assert.assertEquals(20, volBBucketCount);
+
+ Iterator<OzoneBucket> volABucketAIter =
+ volA.listBuckets("bucket-a-");
+ int volABucketACount = 0;
+ while(volABucketAIter.hasNext()) {
+ volABucketAIter.next();
+ volABucketACount++;
+ }
+ Assert.assertEquals(10, volABucketACount);
+ Iterator<OzoneBucket> volBBucketBIter =
+ volA.listBuckets("bucket-b-");
+ int volBBucketBCount = 0;
+ while(volBBucketBIter.hasNext()) {
+ volBBucketBIter.next();
+ volBBucketBCount++;
+ }
+ Assert.assertEquals(10, volBBucketBCount);
+ Iterator<OzoneBucket> volABucketBIter = volA.listBuckets("bucket-b-");
+ for(int i = 0; i < 10; i++) {
+ Assert.assertTrue(volABucketBIter.next().getName()
+ .startsWith(bucketBaseNameB + i + "-"));
+ }
+ Assert.assertFalse(volABucketBIter.hasNext());
+ Iterator<OzoneBucket> volBBucketAIter = volB.listBuckets("bucket-a-");
+ for(int i = 0; i < 10; i++) {
+ Assert.assertTrue(volBBucketAIter.next().getName()
+ .startsWith(bucketBaseNameA + i + "-"));
+ }
+ Assert.assertFalse(volBBucketAIter.hasNext());
+
+ }
+
+ @Test
+ public void listKeyTest()
+ throws IOException, OzoneException {
+ String volumeA = "vol-a-" + RandomStringUtils.randomNumeric(5);
+ String volumeB = "vol-b-" + RandomStringUtils.randomNumeric(5);
+ String bucketA = "buc-a-" + RandomStringUtils.randomNumeric(5);
+ String bucketB = "buc-b-" + RandomStringUtils.randomNumeric(5);
+ store.createVolume(volumeA);
+ store.createVolume(volumeB);
+ OzoneVolume volA = store.getVolume(volumeA);
+ OzoneVolume volB = store.getVolume(volumeB);
+ volA.createBucket(bucketA);
+ volA.createBucket(bucketB);
+ volB.createBucket(bucketA);
+ volB.createBucket(bucketB);
+ OzoneBucket volAbucketA = volA.getBucket(bucketA);
+ OzoneBucket volAbucketB = volA.getBucket(bucketB);
+ OzoneBucket volBbucketA = volB.getBucket(bucketA);
+ OzoneBucket volBbucketB = volB.getBucket(bucketB);
+
+ /*
+ Create 10 keys in vol-a-<random>/buc-a-<random>,
+ vol-a-<random>/buc-b-<random>, vol-b-<random>/buc-a-<random> and
+ vol-b-<random>/buc-b-<random>
+ */
+ String keyBaseA = "key-a-";
+ for (int i = 0; i < 10; i++) {
+ byte[] value = RandomStringUtils.randomAscii(10240).getBytes();
+ OzoneOutputStream one = volAbucketA.createKey(
+ keyBaseA + i + "-" + RandomStringUtils.randomNumeric(5),
+ value.length);
+ one.write(value);
+ one.close();
+ OzoneOutputStream two = volAbucketB.createKey(
+ keyBaseA + i + "-" + RandomStringUtils.randomNumeric(5),
+ value.length);
+ two.write(value);
+ two.close();
+ OzoneOutputStream three = volBbucketA.createKey(
+ keyBaseA + i + "-" + RandomStringUtils.randomNumeric(5),
+ value.length);
+ three.write(value);
+ three.close();
+ OzoneOutputStream four = volBbucketB.createKey(
+ keyBaseA + i + "-" + RandomStringUtils.randomNumeric(5),
+ value.length);
+ four.write(value);
+ four.close();
+ }
+ /*
+ Create 10 keys in vol-a-<random>/buc-a-<random>,
+ vol-a-<random>/buc-b-<random>, vol-b-<random>/buc-a-<random> and
+ vol-b-<random>/buc-b-<random>
+ */
+ String keyBaseB = "key-b-";
+ for (int i = 0; i < 10; i++) {
+ byte[] value = RandomStringUtils.randomAscii(10240).getBytes();
+ OzoneOutputStream one = volAbucketA.createKey(
+ keyBaseB + i + "-" + RandomStringUtils.randomNumeric(5),
+ value.length);
+ one.write(value);
+ one.close();
+ OzoneOutputStream two = volAbucketB.createKey(
+ keyBaseB + i + "-" + RandomStringUtils.randomNumeric(5),
+ value.length);
+ two.write(value);
+ two.close();
+ OzoneOutputStream three = volBbucketA.createKey(
+ keyBaseB + i + "-" + RandomStringUtils.randomNumeric(5),
+ value.length);
+ three.write(value);
+ three.close();
+ OzoneOutputStream four = volBbucketB.createKey(
+ keyBaseB + i + "-" + RandomStringUtils.randomNumeric(5),
+ value.length);
+ four.write(value);
+ four.close();
+ }
+ Iterator<OzoneKey> volABucketAIter =
+ volAbucketA.listKeys("key-");
+ int volABucketAKeyCount = 0;
+ while(volABucketAIter.hasNext()) {
+ volABucketAIter.next();
+ volABucketAKeyCount++;
+ }
+ Assert.assertEquals(20, volABucketAKeyCount);
+ Iterator<OzoneKey> volABucketBIter =
+ volAbucketB.listKeys("key-");
+ int volABucketBKeyCount = 0;
+ while(volABucketBIter.hasNext()) {
+ volABucketBIter.next();
+ volABucketBKeyCount++;
+ }
+ Assert.assertEquals(20, volABucketBKeyCount);
+ Iterator<OzoneKey> volBBucketAIter =
+ volBbucketA.listKeys("key-");
+ int volBBucketAKeyCount = 0;
+ while(volBBucketAIter.hasNext()) {
+ volBBucketAIter.next();
+ volBBucketAKeyCount++;
+ }
+ Assert.assertEquals(20, volBBucketAKeyCount);
+ Iterator<OzoneKey> volBBucketBIter =
+ volBbucketB.listKeys("key-");
+ int volBBucketBKeyCount = 0;
+ while(volBBucketBIter.hasNext()) {
+ volBBucketBIter.next();
+ volBBucketBKeyCount++;
+ }
+ Assert.assertEquals(20, volBBucketBKeyCount);
+ Iterator<OzoneKey> volABucketAKeyAIter =
+ volAbucketA.listKeys("key-a-");
+ int volABucketAKeyACount = 0;
+ while(volABucketAKeyAIter.hasNext()) {
+ volABucketAKeyAIter.next();
+ volABucketAKeyACount++;
+ }
+ Assert.assertEquals(10, volABucketAKeyACount);
+ Iterator<OzoneKey> volABucketAKeyBIter =
+ volAbucketA.listKeys("key-b-");
+ for(int i = 0; i < 10; i++) {
+ Assert.assertTrue(volABucketAKeyBIter.next().getName()
+ .startsWith("key-b-" + i + "-"));
+ }
+ Assert.assertFalse(volABucketBIter.hasNext());
+ }
+
/**
- * Close OzoneClient and shutdown MiniDFSCluster.
+ * Close OzoneClient and shutdown MiniOzoneCluster.
*/
@AfterClass
public static void shutdown() throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org