You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ww...@apache.org on 2017/07/04 01:56:56 UTC
hadoop git commit: HDFS-12035. Ozone: listKey doesn't work from ozone
commandline. Contributed by Yiqun Lin.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 737759e3d -> b23b01645
HDFS-12035. Ozone: listKey doesn't work from ozone commandline. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b23b0164
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b23b0164
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b23b0164
Branch: refs/heads/HDFS-7240
Commit: b23b01645b9374f1741f5cbee6ab3034f4b54760
Parents: 737759e
Author: Weiwei Yang <ww...@apache.org>
Authored: Tue Jul 4 09:56:01 2017 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Tue Jul 4 09:56:01 2017 +0800
----------------------------------------------------------------------
.../hadoop/ozone/web/client/OzoneBucket.java | 21 ++++++-
.../ozone/web/client/OzoneRestClient.java | 19 ++++++-
.../hadoop/ozone/web/interfaces/Bucket.java | 1 +
.../ozone/web/ozShell/keys/ListKeyHandler.java | 18 +++++-
.../hadoop/ozone/web/client/TestKeys.java | 59 ++++++++++++++------
5 files changed, 99 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b0164/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
index cc483ae..53507a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
@@ -41,6 +41,8 @@ import org.apache.http.entity.InputStreamEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
+import com.google.common.base.Strings;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -448,9 +450,14 @@ public class OzoneBucket {
/**
* List all keys in a bucket.
*
+ * @param resultLength The max length of listing result.
+ * @param startKey The start key where to start listing from.
+ * @param prefix The prefix that return list keys start with.
* @return List of OzoneKeys
+ * @throws OzoneException
*/
- public List<OzoneKey> listKeys() throws OzoneException {
+ public List<OzoneKey> listKeys(String resultLength, String startKey,
+ String prefix) throws OzoneException {
HttpGet getRequest = null;
try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
OzoneRestClient client = getVolume().getClient();
@@ -458,6 +465,18 @@ public class OzoneBucket {
builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName())
.build();
+ if (!Strings.isNullOrEmpty(resultLength)) {
+ builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
+ }
+
+ if (!Strings.isNullOrEmpty(startKey)) {
+ builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, startKey);
+ }
+
+ if (!Strings.isNullOrEmpty(prefix)) {
+ builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
+ }
+
getRequest = client.getHttpGet(builder.toString());
return executeListKeys(getRequest, httpClient);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b0164/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
index 9b82222..7c144ad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.web.client;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
@@ -640,10 +641,14 @@ public class OzoneRestClient implements Closeable {
*
* @param volumeName - Volume name
* @param bucketName - Bucket name
+ * @param resultLength The max length of listing result.
+ * @param startKey The start key where to start listing from.
+ * @param prefix The prefix that return list keys start with.
*
* @return List of OzoneKeys
*/
- public List<OzoneKey> listKeys(String volumeName, String bucketName)
+ public List<OzoneKey> listKeys(String volumeName, String bucketName,
+ String resultLength, String startKey, String prefix)
throws OzoneException {
OzoneUtils.verifyResourceName(volumeName);
OzoneUtils.verifyResourceName(bucketName);
@@ -653,6 +658,18 @@ public class OzoneRestClient implements Closeable {
URIBuilder builder = new URIBuilder(getEndPointURI());
builder.setPath("/" + volumeName + "/" + bucketName).build();
+ if (!Strings.isNullOrEmpty(resultLength)) {
+ builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
+ }
+
+ if (!Strings.isNullOrEmpty(startKey)) {
+ builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, startKey);
+ }
+
+ if (!Strings.isNullOrEmpty(prefix)) {
+ builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
+ }
+
getRequest = getHttpGet(builder.toString());
return OzoneBucket.executeListKeys(getRequest, httpClient);
} catch (IOException | URISyntaxException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b0164/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java
index 07e19c1..13866f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java
@@ -126,6 +126,7 @@ public interface Bucket {
String info,
@QueryParam(Header.OZONE_LIST_QUERY_PREFIX)
String prefix,
+ @DefaultValue(Header.OZONE_DEFAULT_LIST_SIZE)
@QueryParam(Header.OZONE_LIST_QUERY_MAXKEYS)
int maxKeys,
@QueryParam(Header.OZONE_LIST_QUERY_PREVKEY)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b0164/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
index 7528189..0e8a773 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
@@ -60,6 +60,21 @@ public class ListKeyHandler extends Handler {
"Incorrect call : listKey is missing");
}
+ String length = null;
+ if (cmd.hasOption(Shell.LIST_LENGTH)) {
+ length = cmd.getOptionValue(Shell.LIST_LENGTH);
+ }
+
+ String startKey = null;
+ if (cmd.hasOption(Shell.START)) {
+ startKey = cmd.getOptionValue(Shell.START);
+ }
+
+ String prefix = null;
+ if (cmd.hasOption(Shell.PREFIX)) {
+ prefix = cmd.getOptionValue(Shell.PREFIX);
+ }
+
String ozoneURIString = cmd.getOptionValue(Shell.LIST_KEY);
URI ozoneURI = verifyURI(ozoneURIString);
Path path = Paths.get(ozoneURI.getPath());
@@ -87,7 +102,8 @@ public class ListKeyHandler extends Handler {
client.setEndPointURI(ozoneURI);
client.setUserAuth(userName);
- List<OzoneKey> keys = client.listKeys(volumeName, bucketName);
+ List<OzoneKey> keys = client.listKeys(volumeName, bucketName, length,
+ startKey, prefix);
for (OzoneKey key : keys) {
System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
key.getObjectInfo().toJsonString()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b0164/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
index bae0422..72800d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.Status;
import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
@@ -82,7 +83,7 @@ public class TestKeys {
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);
cluster = new MiniOzoneCluster.Builder(conf)
- .setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build();
+ .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
DataNode dataNode = cluster.getDataNodes().get(0);
final int port = dataNode.getInfoPort();
client = new OzoneRestClient(String.format("http://localhost:%d", port));
@@ -170,14 +171,14 @@ public class TestKeys {
helper.putKey();
assertNotNull(helper.getBucket());
assertNotNull(helper.getFile());
- List<OzoneKey> keyList = helper.getBucket().listKeys();
+ List<OzoneKey> keyList = helper.getBucket().listKeys("100", null, null);
Assert.assertEquals(keyList.size(), 1);
// test list key using a more efficient call
String newkeyName = OzoneUtils.getRequestID().toLowerCase();
client.putKey(helper.getVol().getVolumeName(),
helper.getBucket().getBucketName(), newkeyName, helper.getFile());
- keyList = helper.getBucket().listKeys();
+ keyList = helper.getBucket().listKeys("100", null, null);
Assert.assertEquals(keyList.size(), 2);
// test new put key with invalid volume/bucket name
@@ -188,7 +189,7 @@ public class TestKeys {
+ " when using invalid volume name.");
} catch(OzoneException e) {
GenericTestUtils.assertExceptionContains(
- ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
+ Status.INTERNAL_ERROR.toString(), e);
}
try {
@@ -198,7 +199,7 @@ public class TestKeys {
+ "when using invalid bucket name.");
} catch (OzoneException e) {
GenericTestUtils.assertExceptionContains(
- ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
+ Status.INTERNAL_ERROR.toString(), e);
}
}
@@ -283,7 +284,7 @@ public class TestKeys {
+ "when using invalid volume name.");
} catch (OzoneException e) {
GenericTestUtils.assertExceptionContains(
- ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
+ Status.KEY_NOT_FOUND.toString(), e);
}
try {
@@ -293,7 +294,7 @@ public class TestKeys {
+ "when using invalid bucket name.");
} catch (OzoneException e) {
GenericTestUtils.assertExceptionContains(
- ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
+ Status.KEY_NOT_FOUND.toString(), e);
}
}
@@ -310,8 +311,8 @@ public class TestKeys {
helper.getBucket().getKey(keyName);
fail("Get Key on a deleted key should have thrown");
} catch (OzoneException ex) {
- assertEquals(ex.getShortMessage(),
- ErrorTable.INVALID_RESOURCE_NAME.getShortMessage());
+ GenericTestUtils.assertExceptionContains(
+ Status.KEY_NOT_FOUND.toString(), ex);
}
}
@@ -322,34 +323,60 @@ public class TestKeys {
assertNotNull(helper.getBucket());
assertNotNull(helper.getFile());
+ // add keys [list-key0, list-key1, ..., list-key9]
for (int x = 0; x < 10; x++) {
- String newkeyName = OzoneUtils.getRequestID().toLowerCase();
+ String newkeyName = "list-key" + x;
helper.getBucket().putKey(newkeyName, helper.getFile());
}
- List<OzoneKey> keyList1 = helper.getBucket().listKeys();
+ List<OzoneKey> keyList1 = helper.getBucket().listKeys("100", null, null);
// test list key using a more efficient call
List<OzoneKey> keyList2 = client.listKeys(helper.getVol().getVolumeName(),
- helper.getBucket().getBucketName());
+ helper.getBucket().getBucketName(), "100", null, null);
Assert.assertEquals(keyList1.size(), 11);
Assert.assertEquals(keyList2.size(), 11);
+ // test maxLength parameter of list keys
+ keyList1 = helper.getBucket().listKeys("1", null, null);
+ keyList2 = client.listKeys(helper.getVol().getVolumeName(),
+ helper.getBucket().getBucketName(), "1", null, null);
+ Assert.assertEquals(keyList1.size(), 1);
+ Assert.assertEquals(keyList2.size(), 1);
+
+ // test startKey parameter of list keys
+ keyList1 = helper.getBucket().listKeys("100", "list-key5", null);
+ keyList2 = client.listKeys(helper.getVol().getVolumeName(),
+ helper.getBucket().getBucketName(), "100", "list-key5", null);
+ Assert.assertEquals(keyList1.size(), 5);
+ Assert.assertEquals(keyList2.size(), 5);
+
+ // test prefix parameter of list keys
+ keyList1 = helper.getBucket().listKeys("100", null, "list-key2");
+ keyList2 = client.listKeys(helper.getVol().getVolumeName(),
+ helper.getBucket().getBucketName(), "100", null, "list-key2");
+ Assert.assertTrue(keyList1.size() == 1
+ && keyList1.get(0).getObjectInfo().getKeyName().equals("list-key2"));
+ Assert.assertTrue(keyList2.size() == 1
+ && keyList2.get(0).getObjectInfo().getKeyName().equals("list-key2"));
+
// test new list keys with invalid volume/bucket name
try {
- client.listKeys("invalid-volume", helper.getBucket().getBucketName());
+ client.listKeys("invalid-volume", helper.getBucket().getBucketName(),
+ "100", null, null);
fail("List keys should have thrown when using invalid volume name.");
} catch (OzoneException e) {
GenericTestUtils.assertExceptionContains(
- ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
+ ErrorTable.SERVER_ERROR.getMessage(), e);
}
try {
- client.listKeys(helper.getVol().getVolumeName(), "invalid-bucket");
+ client.listKeys(helper.getVol().getVolumeName(), "invalid-bucket", "100",
+ null, null);
fail("List keys should have thrown when using invalid bucket name.");
} catch (OzoneException e) {
GenericTestUtils.assertExceptionContains(
- ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
+ ErrorTable.SERVER_ERROR.getMessage(), e);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org