You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2017/07/01 07:15:05 UTC
hadoop git commit: HDFS-11955. Ozone: Set proper parameter default
values for listBuckets http request. Contributed by Weiwei Yang.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 2708e6260 -> f4af0c91c
HDFS-11955. Ozone: Set proper parameter default values for listBuckets http request. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4af0c91
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4af0c91
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4af0c91
Branch: refs/heads/HDFS-7240
Commit: f4af0c91c514316f8114f279479328bfdb19d4b4
Parents: 2708e62
Author: Anu Engineer <ae...@apache.org>
Authored: Sat Jul 1 00:08:30 2017 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Sat Jul 1 00:08:30 2017 -0700
----------------------------------------------------------------------
...ceManagerProtocolServerSideTranslatorPB.java | 8 ++++++-
.../hadoop/ozone/web/client/OzoneVolume.java | 13 +++++++++-
.../hadoop/ozone/web/interfaces/Volume.java | 1 +
.../apache/hadoop/ozone/web/ozShell/Shell.java | 25 +++++++++++++++++++-
.../web/ozShell/bucket/ListBucketHandler.java | 16 ++++++++++++-
.../hadoop/ozone/web/client/TestBuckets.java | 16 ++++++++++---
6 files changed, 72 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4af0c91/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
index d25630c..0b131ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
@@ -81,7 +81,8 @@ import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.List
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.ListKeysResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.Status;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
@@ -93,6 +94,8 @@ import java.util.List;
*/
public class KeySpaceManagerProtocolServerSideTranslatorPB implements
KeySpaceManagerProtocolPB {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(KeySpaceManagerProtocolServerSideTranslatorPB.class);
private final KeySpaceManagerProtocol impl;
/**
@@ -134,6 +137,9 @@ public class KeySpaceManagerProtocolServerSideTranslatorPB implements
return Status.INTERNAL_ERROR;
}
} else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Unknown error occurs", ex);
+ }
return Status.INTERNAL_ERROR;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4af0c91/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
index e1dfec1..91faa04 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.web.client;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.OzoneClientUtils;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
@@ -428,11 +429,21 @@ public class OzoneVolume {
*
* @throws OzoneException
*/
- public List<OzoneBucket> listBuckets() throws OzoneException {
+ public List<OzoneBucket> listBuckets(String resultLength,
+ String startBucket, String prefix) throws OzoneException {
HttpGet getRequest = null;
try (CloseableHttpClient httpClient = newHttpClient()) {
URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
builder.setPath("/" + getVolumeName()).build();
+ if (!Strings.isNullOrEmpty(resultLength)) {
+ builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
+ }
+ if (!Strings.isNullOrEmpty(startBucket)) {
+ builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, startBucket);
+ }
+ if (!Strings.isNullOrEmpty(prefix)) {
+ builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
+ }
getRequest = client.getHttpGet(builder.toString());
return executeListBuckets(getRequest, httpClient);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4af0c91/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
index 8511373..e74ded7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
@@ -136,6 +136,7 @@ public interface Volume {
String info,
@QueryParam(Header.OZONE_LIST_QUERY_PREFIX)
String prefix,
+ @DefaultValue(Header.OZONE_DEFAULT_LIST_SIZE)
@QueryParam(Header.OZONE_LIST_QUERY_MAXKEYS)
int keys,
@QueryParam(Header.OZONE_LIST_QUERY_PREVKEY)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4af0c91/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
index c0d3651..294d8f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
@@ -90,6 +90,11 @@ public class Shell extends Configured implements Tool {
public static final String LIST_KEY = "listKey";
public static final String FILE = "file";
+ // Listing related command line arguments
+ public static final String LIST_LENGTH = "length";
+ public static final String START = "start";
+ public static final String PREFIX = "prefix";
+
/**
* Main for the ozShell Command handling.
*
@@ -136,6 +141,7 @@ public class Shell extends Configured implements Tool {
addVolumeCommands(opts);
addBucketCommands(opts);
addKeyCommands(opts);
+ addListingCommands(opts);
return opts;
}
@@ -249,7 +255,6 @@ public class Shell extends Configured implements Tool {
new Option(REMOVE_ACLS, true, "allows user to remove acls from a " +
"bucket.");
opts.addOption(removeAcl);
-
}
/**
@@ -285,6 +290,24 @@ public class Shell extends Configured implements Tool {
}
/**
+ * Sub commands for list command.
+ * @param opts
+ */
+ private void addListingCommands(Options opts) {
+ Option maxKeys = new Option(LIST_LENGTH, true,
+ "Specify the max length of listing result.");
+ opts.addOption(maxKeys);
+
+ Option prevKey = new Option(START, true,
+ "Specify the start key where to start listing from.");
+ opts.addOption(prevKey);
+
+ Option prefix = new Option(PREFIX, true,
+ "Specify the prefix to filter the listing result.");
+ opts.addOption(prefix);
+ }
+
+ /**
* Dispatches calls to the right command Handler classes.
*
* @param cmd - CommandLine
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4af0c91/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
index 51c8016..ce4a41e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
@@ -82,9 +82,23 @@ public class ListBucketHandler extends Handler {
client.setEndPointURI(ozoneURI);
client.setUserAuth(rootName);
+ String length = null;
+ if (cmd.hasOption(Shell.LIST_LENGTH)) {
+ length = cmd.getOptionValue(Shell.LIST_LENGTH);
+ }
+
+ String startBucket = null;
+ if (cmd.hasOption(Shell.START)) {
+ startBucket = cmd.getOptionValue(Shell.START);
+ }
+
+ String prefix = null;
+ if (cmd.hasOption(Shell.PREFIX)) {
+ prefix = cmd.getOptionValue(Shell.PREFIX);
+ }
OzoneVolume vol = client.getVolume(volumeName);
- List<OzoneBucket> bucketList = vol.listBuckets();
+ List<OzoneBucket> bucketList = vol.listBuckets(length, startBucket, prefix);
for (OzoneBucket bucket : bucketList) {
System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4af0c91/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
index 9ea84eb..635e463 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
@@ -72,7 +72,7 @@ public class TestBuckets {
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
cluster = new MiniOzoneCluster.Builder(conf)
- .setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build();
+ .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
DataNode dataNode = cluster.getDataNodes().get(0);
final int port = dataNode.getInfoPort();
client = new OzoneRestClient(String.format("http://localhost:%d", port));
@@ -168,11 +168,21 @@ public class TestBuckets {
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
String[] acls = {"user:frodo:rw", "user:samwise:rw"};
for (int x = 0; x < 10; x++) {
- String bucketName = OzoneUtils.getRequestID().toLowerCase();
+ String bucketName = "listbucket-test-" + x;
vol.createBucket(bucketName, acls);
}
- List<OzoneBucket> bucketList = vol.listBuckets();
+ List<OzoneBucket> bucketList = vol.listBuckets("100", null, null);
assertEquals(bucketList.size(), 10);
+
+ bucketList = vol.listBuckets("3", null, null);
+ assertEquals(bucketList.size(), 3);
+
+ bucketList = vol.listBuckets("100", "listbucket-test-5", null);
+ assertEquals(bucketList.size(), 5);
+
+ bucketList = vol.listBuckets("100", null, "listbucket-test-3");
+ assertEquals(bucketList.size(), 1);
+
client.close();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org