You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/05/14 17:44:11 UTC
[08/50] [abbrv] hadoop git commit: HDDS-18. Ozone Shell should use
RestClient and RpcClient. Contributed by Lokesh Jain.
HDDS-18. Ozone Shell should use RestClient and RpcClient. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4db209ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4db209ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4db209ba
Branch: refs/heads/HDDS-4
Commit: 4db209baa647c49e19a9ee400ecd9d9901575ed2
Parents: d6d27ce
Author: Anu Engineer <ae...@apache.org>
Authored: Wed May 9 21:23:33 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Mon May 14 10:31:08 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/ozone/OzoneConsts.java | 12 +-
.../src/test/compose/docker-compose.yaml | 1 +
.../test/robotframework/acceptance/ozone.robot | 13 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 49 +--
.../apache/hadoop/ozone/client/OzoneBucket.java | 25 +-
.../ozone/client/OzoneClientException.java | 54 ++++
.../hadoop/ozone/client/OzoneClientFactory.java | 2 +-
.../hadoop/ozone/client/OzoneClientUtils.java | 86 ++++++
.../apache/hadoop/ozone/client/OzoneVolume.java | 26 +-
.../hadoop/ozone/client/rest/RestClient.java | 8 +-
.../hadoop/ozone/web/client/OzoneBucket.java | 47 +--
.../ozone/web/client/OzoneRestClient.java | 43 +--
.../web/client/OzoneRestClientException.java | 54 ----
.../hadoop/ozone/web/client/OzoneVolume.java | 23 +-
.../hadoop/ozone/ozShell/TestOzoneShell.java | 298 +++++++++++++------
.../hadoop/ozone/web/client/TestBuckets.java | 3 +-
.../hadoop/ozone/web/client/TestVolume.java | 3 +-
.../hadoop/ozone/web/ozShell/Handler.java | 59 +++-
.../web/ozShell/bucket/CreateBucketHandler.java | 32 +-
.../web/ozShell/bucket/DeleteBucketHandler.java | 20 +-
.../web/ozShell/bucket/InfoBucketHandler.java | 25 +-
.../web/ozShell/bucket/ListBucketHandler.java | 51 ++--
.../web/ozShell/bucket/UpdateBucketHandler.java | 41 +--
.../web/ozShell/keys/DeleteKeyHandler.java | 26 +-
.../ozone/web/ozShell/keys/GetKeyHandler.java | 46 +--
.../ozone/web/ozShell/keys/InfoKeyHandler.java | 31 +-
.../ozone/web/ozShell/keys/ListKeyHandler.java | 48 +--
.../ozone/web/ozShell/keys/PutKeyHandler.java | 49 ++-
.../web/ozShell/volume/CreateVolumeHandler.java | 28 +-
.../web/ozShell/volume/DeleteVolumeHandler.java | 18 +-
.../web/ozShell/volume/InfoVolumeHandler.java | 38 +--
.../web/ozShell/volume/ListVolumeHandler.java | 58 ++--
.../web/ozShell/volume/UpdateVolumeHandler.java | 24 +-
.../org/apache/hadoop/fs/ozone/Constants.java | 6 -
.../java/org/apache/hadoop/fs/ozone/OzFs.java | 3 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 4 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 3 +-
.../fs/ozone/TestOzoneFileInterfaces.java | 5 +-
.../hadoop/fs/ozone/contract/OzoneContract.java | 6 +-
39 files changed, 796 insertions(+), 572 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 2f9e469..451a08f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -58,6 +58,10 @@ public final class OzoneConsts {
public static final String OZONE_USER = "user";
public static final String OZONE_REQUEST = "request";
+ public static final String OZONE_URI_SCHEME = "o3";
+ public static final String OZONE_HTTP_SCHEME = "http";
+ public static final String OZONE_URI_DELIMITER = "/";
+
public static final String CONTAINER_EXTENSION = ".container";
public static final String CONTAINER_META = ".meta";
@@ -95,7 +99,13 @@ public final class OzoneConsts {
/**
* Supports Bucket Versioning.
*/
- public enum Versioning {NOT_DEFINED, ENABLED, DISABLED}
+ public enum Versioning {
+ NOT_DEFINED, ENABLED, DISABLED;
+
+ public static Versioning getVersioning(boolean versioning) {
+ return versioning ? ENABLED : DISABLED;
+ }
+ }
/**
* Ozone handler types.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml
index 8350eae..da63f84 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml
+++ b/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml
@@ -39,6 +39,7 @@ services:
- ./docker-config
ksm:
image: apache/hadoop-runner
+ hostname: ksm
volumes:
- ${HADOOPDIR}:/opt/hadoop
ports:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot
index ea9131e..1a9cee7 100644
--- a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot
+++ b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot
@@ -49,14 +49,14 @@ Test rest interface
Should contain ${result} 200 OK
Test ozone cli
- Execute on datanode ozone oz -createVolume http://localhost:9880/hive -user bilbo -quota 100TB -root
- ${result} = Execute on datanode ozone oz -listVolume http://localhost:9880/ -user bilbo | grep -v Removed | jq '.[] | select(.volumeName=="hive")'
+ Execute on datanode ozone oz -createVolume http://ksm/hive -user bilbo -quota 100TB -root
+ ${result} = Execute on datanode ozone oz -listVolume o3://ksm -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
Should contain ${result} createdOn
- Execute on datanode ozone oz -createBucket http://localhost:9880/hive/bb1
- ${result} Execute on datanode ozone oz -listBucket http://localhost:9880/hive/ | grep -v Removed | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
+ Execute on datanode ozone oz -createBucket http://ksm/hive/bb1
+ ${result} Execute on datanode ozone oz -listBucket o3://ksm/hive/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
Should Be Equal ${result} hive
- Execute on datanode ozone oz -deleteBucket http://localhost:9880/hive/bb1
- Execute on datanode ozone oz -deleteVolume http://localhost:9880/hive -user bilbo
+ Execute on datanode ozone oz -deleteBucket http://ksm/hive/bb1
+ Execute on datanode ozone oz -deleteVolume http://ksm/hive -user bilbo
@@ -106,6 +106,7 @@ Scale datanodes up
Execute on
[arguments] ${componentname} ${command}
${rc} ${return} = Run docker compose exec ${componentname} ${command}
+ Log ${return}
[return] ${return}
Run docker compose
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index b915213..d8b3011 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -106,25 +106,42 @@ public class ObjectStore {
*/
public Iterator<OzoneVolume> listVolumes(String volumePrefix)
throws IOException {
- return new VolumeIterator(volumePrefix);
+ return listVolumes(volumePrefix, null);
}
/**
- * Returns Iterator to iterate over the List of volumes owned by a specific
- * user. The result can be restricted using volume prefix, will return all
- * volumes if volume prefix is null. If user is null, returns the volume of
- * current user.
+ * Returns Iterator to iterate over all the volumes after prevVolume in object
+ * store. If prevVolume is null it iterates from the first volume.
+ * The result can be restricted using volume prefix, will return all
+ * volumes if volume prefix is null.
+ *
+ * @param volumePrefix Volume prefix to match
+ * @param prevVolume Volumes will be listed after this volume name
+ * @return {@code Iterator<OzoneVolume>}
+ */
+ public Iterator<OzoneVolume> listVolumes(String volumePrefix,
+ String prevVolume) throws IOException {
+ return new VolumeIterator(null, volumePrefix, prevVolume);
+ }
+
+ /**
+ * Returns Iterator to iterate over the list of volumes after prevVolume owned
+ * by a specific user. The result can be restricted using volume prefix, will
+ * return all volumes if volume prefix is null. If user is not null, returns
+ * the volume of current user.
*
* @param user User Name
* @param volumePrefix Volume prefix to match
+ * @param prevVolume Volumes will be listed after this volume name
* @return {@code Iterator<OzoneVolume>}
*/
- public Iterator<OzoneVolume> listVolumes(String user, String volumePrefix)
+ public Iterator<OzoneVolume> listVolumesByUser(String user,
+ String volumePrefix, String prevVolume)
throws IOException {
if(Strings.isNullOrEmpty(user)) {
user = UserGroupInformation.getCurrentUser().getShortUserName();
}
- return new VolumeIterator(user, volumePrefix);
+ return new VolumeIterator(user, volumePrefix, prevVolume);
}
/**
@@ -150,25 +167,17 @@ public class ObjectStore {
private OzoneVolume currentValue;
/**
- * Creates an Iterator to iterate over all volumes in the cluster,
- * which matches the volume prefix.
- * @param volPrefix prefix to match
- */
- VolumeIterator(String volPrefix) {
- this(null, volPrefix);
- }
-
- /**
- * Creates an Iterator to iterate over all volumes of the user,
- * which matches volume prefix.
+ * Creates an Iterator to iterate over all volumes after prevVolume of the user.
+ * If prevVolume is null it iterates from the first volume. The returned volumes
+ * match volume prefix.
* @param user user name
* @param volPrefix volume prefix to match
*/
- VolumeIterator(String user, String volPrefix) {
+ VolumeIterator(String user, String volPrefix, String prevVolume) {
this.user = user;
this.volPrefix = volPrefix;
this.currentValue = null;
- this.currentIterator = getNextListOfVolumes(null).iterator();
+ this.currentIterator = getNextListOfVolumes(prevVolume).iterator();
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index 1712979..5df0254 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -291,7 +291,21 @@ public class OzoneBucket {
* @return {@code Iterator<OzoneKey>}
*/
public Iterator<OzoneKey> listKeys(String keyPrefix) {
- return new KeyIterator(keyPrefix);
+ return listKeys(keyPrefix, null);
+ }
+
+ /**
+ * Returns Iterator to iterate over all keys after prevKey in the bucket.
+ * If prevKey is null it iterates from the first key in the bucket.
+ * The result can be restricted using key prefix, will return all
+ * keys if key prefix is null.
+ *
+ * @param keyPrefix Bucket prefix to match
+ * @param prevKey Keys will be listed after this key name
+ * @return {@code Iterator<OzoneKey>}
+ */
+ public Iterator<OzoneKey> listKeys(String keyPrefix, String prevKey) {
+ return new KeyIterator(keyPrefix, prevKey);
}
/**
@@ -325,14 +339,15 @@ public class OzoneBucket {
/**
- * Creates an Iterator to iterate over all keys in the bucket,
- * which matches volume prefix.
+ * Creates an Iterator to iterate over all keys after prevKey in the bucket.
+ * If prevKey is null it iterates from the first key in the bucket.
+ * The returned keys match key prefix.
* @param keyPrefix
*/
- KeyIterator(String keyPrefix) {
+ KeyIterator(String keyPrefix, String prevKey) {
this.keyPrefix = keyPrefix;
this.currentValue = null;
- this.currentIterator = getNextListOfKeys(null).iterator();
+ this.currentIterator = getNextListOfKeys(prevKey).iterator();
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientException.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientException.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientException.java
new file mode 100644
index 0000000..de3116a
--- /dev/null
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientException.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.client;
+
+import org.apache.hadoop.ozone.client.rest.OzoneException;
+
+/**
+ * This exception is thrown by the Ozone Clients.
+ */
+public class OzoneClientException extends OzoneException {
+ /**
+ * Constructor that allows the shortMessage.
+ *
+ * @param shortMessage Short Message
+ */
+ public OzoneClientException(String shortMessage) {
+ super(0, shortMessage, shortMessage);
+ }
+
+ /**
+ * Constructor that allows a shortMessage and an exception.
+ *
+ * @param shortMessage short message
+ * @param ex exception
+ */
+ public OzoneClientException(String shortMessage, Exception ex) {
+ super(0, shortMessage, shortMessage, ex);
+ }
+
+ /**
+ * Constructor that allows the shortMessage and a longer message.
+ *
+ * @param shortMessage Short Message
+ * @param message long error message
+ */
+ public OzoneClientException(String shortMessage, String message) {
+ super(0, shortMessage, message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
index 6b24f2a..dae94aa 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
@@ -292,7 +292,7 @@ public final class OzoneClientFactory {
return ctor.newInstance(config);
} catch (Exception e) {
final String message = "Couldn't create protocol " + protocolClass;
- LOG.error(message + " exception:" + e);
+ LOG.error(message + " exception: ", e);
if (e.getCause() instanceof IOException) {
throw (IOException) e.getCause();
} else if (e instanceof InvocationTargetException) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
new file mode 100644
index 0000000..5c83d9b
--- /dev/null
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.client;
+
+import org.apache.hadoop.hdds.client.OzoneQuota;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.rest.response.BucketInfo;
+import org.apache.hadoop.ozone.client.rest.response.KeyInfo;
+import org.apache.hadoop.ozone.client.rest.response.VolumeInfo;
+import org.apache.hadoop.ozone.client.rest.response.VolumeOwner;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+
+import static org.apache.hadoop.ozone.web.utils.OzoneUtils.formatTime;
+
+/** A utility class for OzoneClient. */
+public final class OzoneClientUtils {
+
+ private OzoneClientUtils() {}
+
+ /**
+ * Returns a BucketInfo object constructed using fields of the input
+ * OzoneBucket object.
+ *
+ * @param bucket OzoneBucket instance from which BucketInfo object needs to
+ * be created.
+ * @return BucketInfo instance
+ */
+ public static BucketInfo asBucketInfo(OzoneBucket bucket) {
+ BucketInfo bucketInfo =
+ new BucketInfo(bucket.getVolumeName(), bucket.getName());
+ bucketInfo.setCreatedOn(OzoneUtils.formatTime(bucket.getCreationTime()));
+ bucketInfo.setStorageType(bucket.getStorageType());
+ bucketInfo.setVersioning(
+ OzoneConsts.Versioning.getVersioning(bucket.getVersioning()));
+ return bucketInfo;
+ }
+
+ /**
+ * Returns a VolumeInfo object constructed using fields of the input
+ * OzoneVolume object.
+ *
+ * @param volume OzoneVolume instance from which VolumeInfo object needs to
+ * be created.
+ * @return VolumeInfo instance
+ */
+ public static VolumeInfo asVolumeInfo(OzoneVolume volume) {
+ VolumeInfo volumeInfo =
+ new VolumeInfo(volume.getName(), formatTime(volume.getCreationTime()),
+ volume.getOwner());
+ volumeInfo.setQuota(OzoneQuota.getOzoneQuota(volume.getQuota()));
+ volumeInfo.setOwner(new VolumeOwner(volume.getOwner()));
+ return volumeInfo;
+ }
+
+ /**
+ * Returns a KeyInfo object constructed using fields of the input
+ * OzoneKey object.
+ *
+ * @param key OzoneKey instance from which KeyInfo object needs to
+ * be created.
+ * @return KeyInfo instance
+ */
+ public static KeyInfo asKeyInfo(OzoneKey key) {
+ KeyInfo keyInfo = new KeyInfo();
+ keyInfo.setKeyName(key.getName());
+ keyInfo.setCreatedOn(formatTime(key.getCreationTime()));
+ keyInfo.setModifiedOn(formatTime(key.getModificationTime()));
+ keyInfo.setSize(key.getDataSize());
+ return keyInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
index c4e7331..4601f1a 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
@@ -222,7 +222,22 @@ public class OzoneVolume {
* @return {@code Iterator<OzoneBucket>}
*/
public Iterator<OzoneBucket> listBuckets(String bucketPrefix) {
- return new BucketIterator(bucketPrefix);
+ return listBuckets(bucketPrefix, null);
+ }
+
+ /**
+ * Returns Iterator to iterate over all buckets after prevBucket in the volume.
+ * If prevBucket is null it iterates from the first bucket in the volume.
+ * The result can be restricted using bucket prefix, will return all
+ * buckets if bucket prefix is null.
+ *
+ * @param bucketPrefix Bucket prefix to match
+ * @param prevBucket Buckets are listed after this bucket
+ * @return {@code Iterator<OzoneBucket>}
+ */
+ public Iterator<OzoneBucket> listBuckets(String bucketPrefix,
+ String prevBucket) {
+ return new BucketIterator(bucketPrefix, prevBucket);
}
/**
@@ -250,14 +265,15 @@ public class OzoneVolume {
/**
- * Creates an Iterator to iterate over all buckets in the volume,
- * which matches volume prefix.
+ * Creates an Iterator to iterate over all buckets after prevBucket in the volume.
+ * If prevBucket is null it iterates from the first bucket in the volume.
+ * The returned buckets match bucket prefix.
* @param bucketPrefix
*/
- BucketIterator(String bucketPrefix) {
+ public BucketIterator(String bucketPrefix, String prevBucket) {
this.bucketPrefix = bucketPrefix;
this.currentValue = null;
- this.currentIterator = getNextListOfBuckets(null).iterator();
+ this.currentIterator = getNextListOfBuckets(prevBucket).iterator();
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index e9885d1..1fd2091 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.ozone.client.rest.headers.Header;
import org.apache.hadoop.ozone.client.rest.response.BucketInfo;
import org.apache.hadoop.ozone.client.rest.response.KeyInfo;
import org.apache.hadoop.ozone.client.rest.response.VolumeInfo;
-import org.apache.hadoop.ozone.client.rpc.RpcClient;
import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
import org.apache.hadoop.ozone.protocol.proto
@@ -94,7 +93,7 @@ import static java.net.HttpURLConnection.HTTP_OK;
public class RestClient implements ClientProtocol {
private static final String PATH_SEPARATOR = "/";
- private static final Logger LOG = LoggerFactory.getLogger(RpcClient.class);
+ private static final Logger LOG = LoggerFactory.getLogger(RestClient.class);
private final Configuration conf;
private final URI ozoneRestUri;
@@ -195,8 +194,9 @@ public class RestClient implements ClientProtocol {
ServiceInfo restServer = selector.getRestServer(dataNodeInfos);
- return NetUtils.createSocketAddr(restServer.getHostname() + ":" +
- restServer.getPort(ServicePort.Type.HTTP));
+ return NetUtils.createSocketAddr(
+ NetUtils.normalizeHostName(restServer.getHostname()) + ":"
+ + restServer.getPort(ServicePort.Type.HTTP));
} finally {
EntityUtils.consume(entity);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
index e6fe0ec..3183d03 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.client.rest.headers.Header;
import org.apache.hadoop.ozone.OzoneAcl;
@@ -169,11 +170,11 @@ public class OzoneBucket {
*/
public void putKey(String keyName, String data) throws OzoneException {
if ((keyName == null) || keyName.isEmpty()) {
- throw new OzoneRestClientException("Invalid key Name.");
+ throw new OzoneClientException("Invalid key Name.");
}
if (data == null) {
- throw new OzoneRestClientException("Invalid data.");
+ throw new OzoneClientException("Invalid data.");
}
HttpPut putRequest = null;
@@ -195,7 +196,7 @@ public class OzoneBucket {
}
executePutKey(putRequest, httpClient);
} catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
IOUtils.closeStream(is);
releaseConnection(putRequest);
@@ -211,7 +212,7 @@ public class OzoneBucket {
*/
public void putKey(File dataFile) throws OzoneException {
if (dataFile == null) {
- throw new OzoneRestClientException("Invalid file object.");
+ throw new OzoneClientException("Invalid file object.");
}
String keyName = dataFile.getName();
putKey(keyName, dataFile);
@@ -228,11 +229,11 @@ public class OzoneBucket {
throws OzoneException {
if ((keyName == null) || keyName.isEmpty()) {
- throw new OzoneRestClientException("Invalid key Name");
+ throw new OzoneClientException("Invalid key Name");
}
if (file == null) {
- throw new OzoneRestClientException("Invalid data stream");
+ throw new OzoneClientException("Invalid data stream");
}
HttpPut putRequest = null;
@@ -253,7 +254,7 @@ public class OzoneBucket {
executePutKey(putRequest, httpClient);
} catch (IOException | URISyntaxException ex) {
- final OzoneRestClientException orce = new OzoneRestClientException(
+ final OzoneClientException orce = new OzoneClientException(
"Failed to putKey: keyName=" + keyName + ", file=" + file);
orce.initCause(ex);
LOG.trace("", orce);
@@ -285,7 +286,7 @@ public class OzoneBucket {
}
if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
+ throw new OzoneClientException("Unexpected null in http payload");
}
throw OzoneException.parse(EntityUtils.toString(entity));
@@ -306,11 +307,11 @@ public class OzoneBucket {
public void getKey(String keyName, Path downloadTo) throws OzoneException {
if ((keyName == null) || keyName.isEmpty()) {
- throw new OzoneRestClientException("Invalid key Name");
+ throw new OzoneClientException("Invalid key Name");
}
if (downloadTo == null) {
- throw new OzoneRestClientException("Invalid download path");
+ throw new OzoneClientException("Invalid download path");
}
FileOutputStream outPutFile = null;
@@ -326,7 +327,7 @@ public class OzoneBucket {
executeGetKey(getRequest, httpClient, outPutFile);
outPutFile.flush();
} catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
IOUtils.closeStream(outPutFile);
releaseConnection(getRequest);
@@ -343,7 +344,7 @@ public class OzoneBucket {
public String getKey(String keyName) throws OzoneException {
if ((keyName == null) || keyName.isEmpty()) {
- throw new OzoneRestClientException("Invalid key Name");
+ throw new OzoneClientException("Invalid key Name");
}
HttpGet getRequest = null;
@@ -360,7 +361,7 @@ public class OzoneBucket {
executeGetKey(getRequest, httpClient, outPutStream);
return outPutStream.toString(ENCODING_NAME);
} catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
IOUtils.closeStream(outPutStream);
releaseConnection(getRequest);
@@ -394,7 +395,7 @@ public class OzoneBucket {
}
if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
+ throw new OzoneClientException("Unexpected null in http payload");
}
throw OzoneException.parse(EntityUtils.toString(entity));
@@ -414,7 +415,7 @@ public class OzoneBucket {
public void deleteKey(String keyName) throws OzoneException {
if ((keyName == null) || keyName.isEmpty()) {
- throw new OzoneRestClientException("Invalid key Name");
+ throw new OzoneClientException("Invalid key Name");
}
HttpDelete deleteRequest = null;
@@ -427,7 +428,7 @@ public class OzoneBucket {
.getClient().getHttpDelete(builder.toString());
executeDeleteKey(deleteRequest, httpClient);
} catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
releaseConnection(deleteRequest);
}
@@ -457,7 +458,7 @@ public class OzoneBucket {
}
if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
+ throw new OzoneClientException("Unexpected null in http payload");
}
throw OzoneException.parse(EntityUtils.toString(entity));
@@ -505,7 +506,7 @@ public class OzoneBucket {
return executeListKeys(getRequest, httpClient);
} catch (IOException | URISyntaxException e) {
- throw new OzoneRestClientException(e.getMessage(), e);
+ throw new OzoneClientException(e.getMessage(), e);
} finally {
releaseConnection(getRequest);
}
@@ -534,7 +535,7 @@ public class OzoneBucket {
getRequest = client.getHttpGet(uri.toString());
return executeListKeys(getRequest, HttpClientBuilder.create().build());
} catch (IOException | URISyntaxException e) {
- throw new OzoneRestClientException(e.getMessage());
+ throw new OzoneClientException(e.getMessage());
} finally {
releaseConnection(getRequest);
}
@@ -560,7 +561,7 @@ public class OzoneBucket {
entity = response.getEntity();
if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
+ throw new OzoneClientException("Unexpected null in http payload");
}
if (errorCode == HTTP_OK) {
String temp = EntityUtils.toString(entity);
@@ -586,7 +587,7 @@ public class OzoneBucket {
*/
public OzoneKey getKeyInfo(String keyName) throws OzoneException {
if ((keyName == null) || keyName.isEmpty()) {
- throw new OzoneRestClientException(
+ throw new OzoneClientException(
"Unable to get key info, key name is null or empty");
}
@@ -604,7 +605,7 @@ public class OzoneBucket {
getRequest = client.getHttpGet(builder.toString());
return executeGetKeyInfo(getRequest, httpClient);
} catch (IOException | URISyntaxException e) {
- throw new OzoneRestClientException(e.getMessage(), e);
+ throw new OzoneClientException(e.getMessage(), e);
} finally {
releaseConnection(getRequest);
}
@@ -627,7 +628,7 @@ public class OzoneBucket {
int errorCode = response.getStatusLine().getStatusCode();
entity = response.getEntity();
if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
+ throw new OzoneClientException("Unexpected null in http payload");
}
if (errorCode == HTTP_OK) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
index 6d0bbf4..8373f67 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
@@ -24,6 +24,7 @@ import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.client.rest.headers.Header;
import org.apache.hadoop.ozone.web.response.ListVolumes;
@@ -111,7 +112,7 @@ public class OzoneRestClient implements Closeable {
*/
public void setEndPointURI(URI endPointURI) throws OzoneException {
if ((endPointURI == null) || (endPointURI.toString().isEmpty())) {
- throw new OzoneRestClientException("Invalid ozone URI");
+ throw new OzoneClientException("Invalid ozone URI");
}
this.endPointURI = endPointURI;
}
@@ -151,7 +152,7 @@ public class OzoneRestClient implements Closeable {
* @param onBehalfOf - The user on behalf we are making the call for
* @param quota - Quota's are specified in a specific format. it is
* integer(MB|GB|TB), for example 100TB.
- * @throws OzoneRestClientException
+ * @throws OzoneClientException
*/
public OzoneVolume createVolume(String volumeName, String onBehalfOf,
String quota) throws OzoneException {
@@ -169,7 +170,7 @@ public class OzoneRestClient implements Closeable {
executeCreateVolume(httpPost, httpClient);
return getVolume(volumeName);
} catch (IOException | URISyntaxException | IllegalArgumentException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
releaseConnection(httpPost);
}
@@ -196,7 +197,7 @@ public class OzoneRestClient implements Closeable {
httpGet = getHttpGet(builder.toString());
return executeInfoVolume(httpGet, httpClient);
} catch (IOException | URISyntaxException | IllegalArgumentException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
releaseConnection(httpGet);
}
@@ -247,7 +248,7 @@ public class OzoneRestClient implements Closeable {
}
return executeListVolume(httpGet, httpClient);
} catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
releaseConnection(httpGet);
}
@@ -329,7 +330,7 @@ public class OzoneRestClient implements Closeable {
return executeListVolume(httpGet, httpClient);
} catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
releaseConnection(httpGet);
}
@@ -351,7 +352,7 @@ public class OzoneRestClient implements Closeable {
httpDelete = getHttpDelete(builder.toString());
executeDeleteVolume(httpDelete, httpClient);
} catch (IOException | URISyntaxException | IllegalArgumentException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
releaseConnection(httpDelete);
}
@@ -368,7 +369,7 @@ public class OzoneRestClient implements Closeable {
throws OzoneException {
HttpPut putRequest = null;
if (newOwner == null || newOwner.isEmpty()) {
- throw new OzoneRestClientException("Invalid new owner name");
+ throw new OzoneClientException("Invalid new owner name");
}
try (CloseableHttpClient httpClient = newHttpClient()) {
OzoneUtils.verifyResourceName(volumeName);
@@ -380,7 +381,7 @@ public class OzoneRestClient implements Closeable {
executePutVolume(putRequest, httpClient);
} catch (URISyntaxException | IllegalArgumentException | IOException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
releaseConnection(putRequest);
}
@@ -399,7 +400,7 @@ public class OzoneRestClient implements Closeable {
public void setVolumeQuota(String volumeName, String quota)
throws OzoneException {
if (quota == null || quota.isEmpty()) {
- throw new OzoneRestClientException("Invalid quota");
+ throw new OzoneClientException("Invalid quota");
}
HttpPut putRequest = null;
try (CloseableHttpClient httpClient = newHttpClient()) {
@@ -413,7 +414,7 @@ public class OzoneRestClient implements Closeable {
executePutVolume(putRequest, httpClient);
} catch (URISyntaxException | IllegalArgumentException | IOException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
releaseConnection(putRequest);
}
@@ -443,7 +444,7 @@ public class OzoneRestClient implements Closeable {
if (entity != null) {
throw OzoneException.parse(EntityUtils.toString(entity));
} else {
- throw new OzoneRestClientException("Unexpected null in http payload");
+ throw new OzoneClientException("Unexpected null in http payload");
}
} finally {
if (entity != null) {
@@ -470,7 +471,7 @@ public class OzoneRestClient implements Closeable {
entity = response.getEntity();
if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
+ throw new OzoneClientException("Unexpected null in http payload");
}
if (errorCode == HTTP_OK) {
@@ -531,7 +532,7 @@ public class OzoneRestClient implements Closeable {
entity = response.getEntity();
if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
+ throw new OzoneClientException("Unexpected null in http payload");
}
String temp = EntityUtils.toString(entity);
@@ -595,11 +596,11 @@ public class OzoneRestClient implements Closeable {
OzoneUtils.verifyResourceName(bucketName);
if (StringUtils.isEmpty(keyName)) {
- throw new OzoneRestClientException("Invalid key Name");
+ throw new OzoneClientException("Invalid key Name");
}
if (file == null) {
- throw new OzoneRestClientException("Invalid data stream");
+ throw new OzoneClientException("Invalid data stream");
}
HttpPut putRequest = null;
@@ -619,7 +620,7 @@ public class OzoneRestClient implements Closeable {
putRequest.setHeader(Header.CONTENT_MD5, DigestUtils.md5Hex(fis));
OzoneBucket.executePutKey(putRequest, httpClient);
} catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
IOUtils.closeStream(fis);
releaseConnection(putRequest);
@@ -641,11 +642,11 @@ public class OzoneRestClient implements Closeable {
OzoneUtils.verifyResourceName(bucketName);
if (StringUtils.isEmpty(keyName)) {
- throw new OzoneRestClientException("Invalid key Name");
+ throw new OzoneClientException("Invalid key Name");
}
if (downloadTo == null) {
- throw new OzoneRestClientException("Invalid download path");
+ throw new OzoneClientException("Invalid download path");
}
FileOutputStream outPutFile = null;
@@ -661,7 +662,7 @@ public class OzoneRestClient implements Closeable {
OzoneBucket.executeGetKey(getRequest, httpClient, outPutFile);
outPutFile.flush();
} catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
IOUtils.closeStream(outPutFile);
releaseConnection(getRequest);
@@ -706,7 +707,7 @@ public class OzoneRestClient implements Closeable {
getRequest = getHttpGet(builder.toString());
return OzoneBucket.executeListKeys(getRequest, httpClient);
} catch (IOException | URISyntaxException e) {
- throw new OzoneRestClientException(e.getMessage(), e);
+ throw new OzoneClientException(e.getMessage(), e);
} finally {
releaseConnection(getRequest);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java
deleted file mode 100644
index dfb2357..0000000
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.client;
-
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-
-/**
- * This exception is thrown by the Ozone Clients.
- */
-public class OzoneRestClientException extends OzoneException {
- /**
- * Constructor that allows the shortMessage.
- *
- * @param shortMessage Short Message
- */
- public OzoneRestClientException(String shortMessage) {
- super(0, shortMessage, shortMessage);
- }
-
- /**
- * Constructor that allows a shortMessage and an exception.
- *
- * @param shortMessage short message
- * @param ex exception
- */
- public OzoneRestClientException(String shortMessage, Exception ex) {
- super(0, shortMessage, shortMessage, ex);
- }
-
- /**
- * Constructor that allows the shortMessage and a longer message.
- *
- * @param shortMessage Short Message
- * @param message long error message
- */
- public OzoneRestClientException(String shortMessage, String message) {
- super(0, shortMessage, message);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
index 6728e68..9d3831c 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
@@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
+import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.client.rest.headers.Header;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
@@ -203,7 +204,7 @@ public class OzoneVolume {
executeCreateBucket(httpPost, httpClient);
return getBucket(bucketName);
} catch (IOException | URISyntaxException | IllegalArgumentException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
releaseConnection(httpPost);
}
@@ -276,7 +277,7 @@ public class OzoneVolume {
if (entity != null) {
throw OzoneException.parse(EntityUtils.toString(entity));
} else {
- throw new OzoneRestClientException("Unexpected null in http payload");
+ throw new OzoneClientException("Unexpected null in http payload");
}
} finally {
if (entity != null) {
@@ -307,7 +308,7 @@ public class OzoneVolume {
}
executePutBucket(putRequest, httpClient);
} catch (URISyntaxException | IOException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
releaseConnection(putRequest);
}
@@ -336,7 +337,7 @@ public class OzoneVolume {
}
executePutBucket(putRequest, httpClient);
} catch (URISyntaxException | IOException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
releaseConnection(putRequest);
}
@@ -361,7 +362,7 @@ public class OzoneVolume {
return executeInfoBucket(getRequest, httpClient);
} catch (IOException | URISyntaxException | IllegalArgumentException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
releaseConnection(getRequest);
}
@@ -388,7 +389,7 @@ public class OzoneVolume {
int errorCode = response.getStatusLine().getStatusCode();
entity = response.getEntity();
if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
+ throw new OzoneClientException("Unexpected null in http payload");
}
if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
OzoneBucket bucket =
@@ -432,7 +433,7 @@ public class OzoneVolume {
throw OzoneException.parse(EntityUtils.toString(entity));
}
- throw new OzoneRestClientException("Unexpected null in http result");
+ throw new OzoneClientException("Unexpected null in http result");
} finally {
if (entity != null) {
EntityUtils.consumeQuietly(entity);
@@ -467,7 +468,7 @@ public class OzoneVolume {
return executeListBuckets(getRequest, httpClient);
} catch (IOException | URISyntaxException e) {
- throw new OzoneRestClientException(e.getMessage(), e);
+ throw new OzoneClientException(e.getMessage(), e);
} finally {
releaseConnection(getRequest);
}
@@ -496,7 +497,7 @@ public class OzoneVolume {
entity = response.getEntity();
if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
+ throw new OzoneClientException("Unexpected null in http payload");
}
if (errorCode == HTTP_OK) {
ListBuckets bucketList =
@@ -535,7 +536,7 @@ public class OzoneVolume {
executeDeleteBucket(delRequest, httpClient);
} catch (IOException | URISyntaxException | IllegalArgumentException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
+ throw new OzoneClientException(ex.getMessage(), ex);
} finally {
releaseConnection(delRequest);
}
@@ -564,7 +565,7 @@ public class OzoneVolume {
}
if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload.");
+ throw new OzoneClientException("Unexpected null in http payload.");
}
throw OzoneException.parse(EntityUtils.toString(entity));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
index 31046f5..8ffe67d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.ozShell;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -26,15 +27,21 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
+import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.UUID;
+import java.util.stream.Collectors;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneAcl;
@@ -43,11 +50,16 @@ import org.apache.hadoop.ozone.OzoneAcl.OzoneACLType;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneKey;
-import org.apache.hadoop.ozone.web.client.OzoneRestClient;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.VolumeArgs;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.client.rest.OzoneException;
+import org.apache.hadoop.ozone.client.rest.RestClient;
+import org.apache.hadoop.ozone.client.rpc.RpcClient;
+import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
import org.apache.hadoop.ozone.web.ozShell.Shell;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.response.BucketInfo;
@@ -63,12 +75,20 @@ import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This test class specified for testing Ozone shell command.
*/
+@RunWith(value = Parameterized.class)
public class TestOzoneShell {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestOzoneShell.class);
+
/**
* Set the timeout for every test.
*/
@@ -79,7 +99,7 @@ public class TestOzoneShell {
private static File baseDir;
private static OzoneConfiguration conf = null;
private static MiniOzoneCluster cluster = null;
- private static OzoneRestClient client = null;
+ private static ClientProtocol client = null;
private static Shell shell = null;
private final ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -87,6 +107,16 @@ public class TestOzoneShell {
private static final PrintStream OLD_OUT = System.out;
private static final PrintStream OLD_ERR = System.err;
+ @Parameterized.Parameters
+ public static Collection<Object[]> clientProtocol() {
+ Object[][] params = new Object[][] {
+ {RpcClient.class},
+ {RestClient.class}};
+ return Arrays.asList(params);
+ }
+
+ @Parameterized.Parameter
+ public Class clientProtocol;
/**
* Create a MiniDFSCluster for testing with using distributed Ozone
* handler type.
@@ -110,13 +140,12 @@ public class TestOzoneShell {
shell = new Shell();
shell.setConf(conf);
- cluster = MiniOzoneCluster.newBuilder(conf).build();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ conf.setInt(OZONE_REPLICATION, ReplicationFactor.THREE.getValue());
+ client = new RpcClient(conf);
cluster.waitForClusterToBeReady();
- final int port = cluster.getHddsDatanodes().get(0).getDatanodeDetails()
- .getOzoneRestPort();
- url = String.format("http://localhost:%d", port);
- client = new OzoneRestClient(String.format("http://localhost:%d", port));
- client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
}
/**
@@ -137,6 +166,26 @@ public class TestOzoneShell {
public void setup() {
System.setOut(new PrintStream(out));
System.setErr(new PrintStream(err));
+ if(clientProtocol.equals(RestClient.class)) {
+ String hostName = cluster.getKeySpaceManager().getHttpServer()
+ .getHttpAddress().getHostName();
+ int port = cluster
+ .getKeySpaceManager().getHttpServer().getHttpAddress().getPort();
+ url = String.format("http://" + hostName + ":" + port);
+ } else {
+ List<ServiceInfo> services = null;
+ try {
+ services = cluster.getKeySpaceManager().getServiceList();
+ } catch (IOException e) {
+ LOG.error("Could not get service list from KSM");
+ }
+ String hostName = services.stream().filter(
+ a -> a.getNodeType().equals(HddsProtos.NodeType.KSM))
+ .collect(Collectors.toList()).get(0).getHostname();
+
+ String port = cluster.getKeySpaceManager().getRpcPort();
+ url = String.format("o3://" + hostName + ":" + port);
+ }
}
@After
@@ -152,22 +201,29 @@ public class TestOzoneShell {
@Test
public void testCreateVolume() throws Exception {
+ LOG.info("Running testCreateVolume");
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
String userName = "bilbo";
String[] args = new String[] {"-createVolume", url + "/" + volumeName,
"-user", userName, "-root"};
assertEquals(0, ToolRunner.run(shell, args));
- OzoneVolume volumeInfo = client.getVolume(volumeName);
- assertEquals(volumeName, volumeInfo.getVolumeName());
- assertEquals(userName, volumeInfo.getOwnerName());
+ OzoneVolume volumeInfo = client.getVolumeDetails(volumeName);
+ assertEquals(volumeName, volumeInfo.getName());
+ assertEquals(userName, volumeInfo.getOwner());
}
@Test
public void testDeleteVolume() throws Exception {
+ LOG.info("Running testDeleteVolume");
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
- assertNotNull(vol);
+ VolumeArgs volumeArgs = VolumeArgs.newBuilder()
+ .setOwner("bilbo")
+ .setQuota("100TB")
+ .build();
+ client.createVolume(volumeName, volumeArgs);
+ OzoneVolume volume = client.getVolumeDetails(volumeName);
+ assertNotNull(volume);
String[] args = new String[] {"-deleteVolume", url + "/" + volumeName,
"-root"};
@@ -175,9 +231,9 @@ public class TestOzoneShell {
// verify if volume has been deleted
try {
- client.getVolume(volumeName);
+ client.getVolumeDetails(volumeName);
fail("Get volume call should have thrown.");
- } catch (OzoneException e) {
+ } catch (IOException e) {
GenericTestUtils.assertExceptionContains(
"Info Volume failed, error:VOLUME_NOT_FOUND", e);
}
@@ -185,8 +241,13 @@ public class TestOzoneShell {
@Test
public void testInfoVolume() throws Exception {
+ LOG.info("Running testInfoVolume");
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- client.createVolume(volumeName, "bilbo", "100TB");
+ VolumeArgs volumeArgs = VolumeArgs.newBuilder()
+ .setOwner("bilbo")
+ .setQuota("100TB")
+ .build();
+ client.createVolume(volumeName, volumeArgs);
String[] args = new String[] {"-infoVolume", url + "/" + volumeName,
"-root"};
@@ -206,45 +267,53 @@ public class TestOzoneShell {
@Test
public void testUpdateVolume() throws Exception {
+ LOG.info("Running testUpdateVolume");
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
String userName = "bilbo";
- OzoneVolume vol = client.createVolume(volumeName, userName, "100TB");
- assertEquals(userName, vol.getOwnerName());
- assertEquals(100, vol.getQuota().getSize(), 100);
- assertEquals(OzoneQuota.Units.TB, vol.getQuota().getUnit());
+ VolumeArgs volumeArgs = VolumeArgs.newBuilder()
+ .setOwner("bilbo")
+ .setQuota("100TB")
+ .build();
+ client.createVolume(volumeName, volumeArgs);
+ OzoneVolume vol = client.getVolumeDetails(volumeName);
+ assertEquals(userName, vol.getOwner());
+ assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(), vol.getQuota());
String[] args = new String[] {"-updateVolume", url + "/" + volumeName,
"-quota", "500MB", "-root"};
assertEquals(0, ToolRunner.run(shell, args));
- vol = client.getVolume(volumeName);
- assertEquals(userName, vol.getOwnerName());
- assertEquals(500, vol.getQuota().getSize(), 500);
- assertEquals(OzoneQuota.Units.MB, vol.getQuota().getUnit());
+ vol = client.getVolumeDetails(volumeName);
+ assertEquals(userName, vol.getOwner());
+ assertEquals(OzoneQuota.parseQuota("500MB").sizeInBytes(), vol.getQuota());
String newUser = "new-user";
args = new String[] {"-updateVolume", url + "/" + volumeName,
"-user", newUser, "-root"};
assertEquals(0, ToolRunner.run(shell, args));
- vol = client.getVolume(volumeName);
- assertEquals(newUser, vol.getOwnerName());
+ vol = client.getVolumeDetails(volumeName);
+ assertEquals(newUser, vol.getOwner());
// test error conditions
args = new String[] {"-updateVolume", url + "/invalid-volume",
"-user", newUser, "-root"};
assertEquals(1, ToolRunner.run(shell, args));
assertTrue(err.toString().contains(
- "Volume owner change failed, error:VOLUME_NOT_FOUND"));
+ "Info Volume failed, error:VOLUME_NOT_FOUND"));
err.reset();
args = new String[] {"-updateVolume", url + "/invalid-volume",
"-quota", "500MB", "-root"};
assertEquals(1, ToolRunner.run(shell, args));
assertTrue(err.toString().contains(
- "Volume quota change failed, error:VOLUME_NOT_FOUND"));
+ "Info Volume failed, error:VOLUME_NOT_FOUND"));
}
@Test
public void testListVolume() throws Exception {
+ LOG.info("Running testListVolume");
+ if (clientProtocol.equals(RestClient.class)) {
+ return;
+ }
String commandOutput;
List<VolumeInfo> volumes;
final int volCount = 20;
@@ -265,7 +334,12 @@ public class TestOzoneShell {
userName = user2;
volumeName = "test-vol" + x;
}
- OzoneVolume vol = client.createVolume(volumeName, userName, "100TB");
+ VolumeArgs volumeArgs = VolumeArgs.newBuilder()
+ .setOwner(userName)
+ .setQuota("100TB")
+ .build();
+ client.createVolume(volumeName, volumeArgs);
+ OzoneVolume vol = client.getVolumeDetails(volumeName);
assertNotNull(vol);
}
@@ -343,16 +417,17 @@ public class TestOzoneShell {
@Test
public void testCreateBucket() throws Exception {
+ LOG.info("Running testCreateBucket");
OzoneVolume vol = creatVolume();
String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
String[] args = new String[] {"-createBucket",
- url + "/" + vol.getVolumeName() + "/" + bucketName};
+ url + "/" + vol.getName() + "/" + bucketName};
assertEquals(0, ToolRunner.run(shell, args));
OzoneBucket bucketInfo = vol.getBucket(bucketName);
- assertEquals(vol.getVolumeName(),
- bucketInfo.getBucketInfo().getVolumeName());
- assertEquals(bucketName, bucketInfo.getBucketName());
+ assertEquals(vol.getName(),
+ bucketInfo.getVolumeName());
+ assertEquals(bucketName, bucketInfo.getName());
// test create a bucket in a non-exist volume
args = new String[] {"-createBucket",
@@ -365,20 +440,22 @@ public class TestOzoneShell {
@Test
public void testDeleteBucket() throws Exception {
+ LOG.info("Running testDeleteBucket");
OzoneVolume vol = creatVolume();
String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- OzoneBucket bucketInfo = vol.createBucket(bucketName);
+ vol.createBucket(bucketName);
+ OzoneBucket bucketInfo = vol.getBucket(bucketName);
assertNotNull(bucketInfo);
String[] args = new String[] {"-deleteBucket",
- url + "/" + vol.getVolumeName() + "/" + bucketName};
+ url + "/" + vol.getName() + "/" + bucketName};
assertEquals(0, ToolRunner.run(shell, args));
// verify if bucket has been deleted in volume
try {
vol.getBucket(bucketName);
fail("Get bucket should have thrown.");
- } catch (OzoneException e) {
+ } catch (IOException e) {
GenericTestUtils.assertExceptionContains(
"Info Bucket failed, error: BUCKET_NOT_FOUND", e);
}
@@ -393,7 +470,7 @@ public class TestOzoneShell {
err.reset();
// test delete non-exist bucket
args = new String[] {"-deleteBucket",
- url + "/" + vol.getVolumeName() + "/invalid-bucket"};
+ url + "/" + vol.getName() + "/invalid-bucket"};
assertEquals(1, ToolRunner.run(shell, args));
assertTrue(err.toString().contains(
"Delete Bucket failed, error:BUCKET_NOT_FOUND"));
@@ -401,12 +478,13 @@ public class TestOzoneShell {
@Test
public void testInfoBucket() throws Exception {
+ LOG.info("Running testInfoBucket");
OzoneVolume vol = creatVolume();
String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
vol.createBucket(bucketName);
String[] args = new String[] {"-infoBucket",
- url + "/" + vol.getVolumeName() + "/" + bucketName};
+ url + "/" + vol.getName() + "/" + bucketName};
assertEquals(0, ToolRunner.run(shell, args));
String output = out.toString();
@@ -416,7 +494,7 @@ public class TestOzoneShell {
// test get info from a non-exist bucket
args = new String[] {"-infoBucket",
- url + "/" + vol.getVolumeName() + "/invalid-bucket" + bucketName};
+ url + "/" + vol.getName() + "/invalid-bucket" + bucketName};
assertEquals(1, ToolRunner.run(shell, args));
assertTrue(err.toString().contains(
"Info Bucket failed, error: BUCKET_NOT_FOUND"));
@@ -424,13 +502,15 @@ public class TestOzoneShell {
@Test
public void testUpdateBucket() throws Exception {
+ LOG.info("Running testUpdateBucket");
OzoneVolume vol = creatVolume();
String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- OzoneBucket bucket = vol.createBucket(bucketName);
- assertEquals(0, bucket.getAcls().size());
+ vol.createBucket(bucketName);
+ OzoneBucket bucket = vol.getBucket(bucketName);
+ int aclSize = bucket.getAcls().size();
String[] args = new String[] {"-updateBucket",
- url + "/" + vol.getVolumeName() + "/" + bucketName, "-addAcl",
+ url + "/" + vol.getName() + "/" + bucketName, "-addAcl",
"user:frodo:rw,group:samwise:r"};
assertEquals(0, ToolRunner.run(shell, args));
String output = out.toString();
@@ -438,36 +518,40 @@ public class TestOzoneShell {
&& output.contains(OzoneConsts.OZONE_TIME_ZONE));
bucket = vol.getBucket(bucketName);
- assertEquals(2, bucket.getAcls().size());
+ assertEquals(2 + aclSize, bucket.getAcls().size());
- OzoneAcl acl = bucket.getAcls().get(0);
+ OzoneAcl acl = bucket.getAcls().get(aclSize);
assertTrue(acl.getName().equals("frodo")
&& acl.getType() == OzoneACLType.USER
&& acl.getRights()== OzoneACLRights.READ_WRITE);
args = new String[] {"-updateBucket",
- url + "/" + vol.getVolumeName() + "/" + bucketName, "-removeAcl",
+ url + "/" + vol.getName() + "/" + bucketName, "-removeAcl",
"user:frodo:rw"};
assertEquals(0, ToolRunner.run(shell, args));
bucket = vol.getBucket(bucketName);
- acl = bucket.getAcls().get(0);
- assertEquals(1, bucket.getAcls().size());
+ acl = bucket.getAcls().get(aclSize);
+ assertEquals(1 + aclSize, bucket.getAcls().size());
assertTrue(acl.getName().equals("samwise")
&& acl.getType() == OzoneACLType.GROUP
&& acl.getRights()== OzoneACLRights.READ);
// test update bucket for a non-exist bucket
args = new String[] {"-updateBucket",
- url + "/" + vol.getVolumeName() + "/invalid-bucket", "-addAcl",
+ url + "/" + vol.getName() + "/invalid-bucket", "-addAcl",
"user:frodo:rw"};
assertEquals(1, ToolRunner.run(shell, args));
assertTrue(err.toString().contains(
- "Setting bucket property failed, error: BUCKET_NOT_FOUND"));
+ "Info Bucket failed, error: BUCKET_NOT_FOUND"));
}
@Test
public void testListBucket() throws Exception {
+ LOG.info("Running testListBucket");
+ if (clientProtocol.equals(RestClient.class)) {
+ return;
+ }
List<BucketInfo> buckets;
String commandOutput;
int bucketCount = 11;
@@ -478,13 +562,14 @@ public class TestOzoneShell {
for (int i = 0; i < bucketCount; i++) {
String name = "test-bucket" + i;
bucketNames.add(name);
- OzoneBucket bucket = vol.createBucket(name);
+ vol.createBucket(name);
+ OzoneBucket bucket = vol.getBucket(name);
assertNotNull(bucket);
}
// test -length option
String[] args = new String[] {"-listBucket",
- url + "/" + vol.getVolumeName(), "-length", "100"};
+ url + "/" + vol.getName(), "-length", "100"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
buckets = (List<BucketInfo>) JsonUtils.toJsonList(commandOutput,
@@ -497,13 +582,13 @@ public class TestOzoneShell {
// test-bucket10, test-bucket2, ,..., test-bucket9]
for (int i = 0; i < buckets.size(); i++) {
assertEquals(buckets.get(i).getBucketName(), bucketNames.get(i));
- assertEquals(buckets.get(i).getVolumeName(), vol.getVolumeName());
+ assertEquals(buckets.get(i).getVolumeName(), vol.getName());
assertTrue(buckets.get(i).getCreatedOn()
.contains(OzoneConsts.OZONE_TIME_ZONE));
}
out.reset();
- args = new String[] {"-listBucket", url + "/" + vol.getVolumeName(),
+ args = new String[] {"-listBucket", url + "/" + vol.getName(),
"-length", "3"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
@@ -519,7 +604,7 @@ public class TestOzoneShell {
// test -prefix option
out.reset();
- args = new String[] {"-listBucket", url + "/" + vol.getVolumeName(),
+ args = new String[] {"-listBucket", url + "/" + vol.getName(),
"-length", "100", "-prefix", "test-bucket1"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
@@ -533,7 +618,7 @@ public class TestOzoneShell {
// test -start option
out.reset();
- args = new String[] {"-listBucket", url + "/" + vol.getVolumeName(),
+ args = new String[] {"-listBucket", url + "/" + vol.getName(),
"-length", "100", "-start", "test-bucket7"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
@@ -546,7 +631,7 @@ public class TestOzoneShell {
// test error conditions
err.reset();
- args = new String[] {"-listBucket", url + "/" + vol.getVolumeName(),
+ args = new String[] {"-listBucket", url + "/" + vol.getName(),
"-length", "-1"};
assertEquals(1, ToolRunner.run(shell, args));
assertTrue(err.toString().contains(
@@ -555,9 +640,10 @@ public class TestOzoneShell {
@Test
public void testPutKey() throws Exception {
+ LOG.info("Running testPutKey");
OzoneBucket bucket = creatBucket();
- String volumeName = bucket.getBucketInfo().getVolumeName();
- String bucketName = bucket.getBucketName();
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getName();
String keyName = "key" + RandomStringUtils.randomNumeric(5);
String[] args = new String[] {"-putKey",
@@ -565,8 +651,8 @@ public class TestOzoneShell {
createTmpFile()};
assertEquals(0, ToolRunner.run(shell, args));
- OzoneKey keyInfo = bucket.getKeyInfo(keyName);
- assertEquals(keyName, keyInfo.getObjectInfo().getKeyName());
+ OzoneKey keyInfo = bucket.getKey(keyName);
+ assertEquals(keyName, keyInfo.getName());
// test put key in a non-exist bucket
args = new String[] {"-putKey",
@@ -574,18 +660,22 @@ public class TestOzoneShell {
createTmpFile()};
assertEquals(1, ToolRunner.run(shell, args));
assertTrue(err.toString().contains(
- "Create key failed, error:BUCKET_NOT_FOUND"));
+ "Info Bucket failed, error: BUCKET_NOT_FOUND"));
}
@Test
public void testGetKey() throws Exception {
+ LOG.info("Running testGetKey");
String keyName = "key" + RandomStringUtils.randomNumeric(5);
OzoneBucket bucket = creatBucket();
- String volumeName = bucket.getBucketInfo().getVolumeName();
- String bucketName = bucket.getBucketName();
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getName();
String dataStr = "test-data";
- bucket.putKey(keyName, dataStr);
+ OzoneOutputStream keyOutputStream =
+ bucket.createKey(keyName, dataStr.length());
+ keyOutputStream.write(dataStr.getBytes());
+ keyOutputStream.close();
String tmpPath = baseDir.getAbsolutePath() + "/testfile-"
+ UUID.randomUUID().toString();
@@ -603,14 +693,19 @@ public class TestOzoneShell {
@Test
public void testDeleteKey() throws Exception {
+ LOG.info("Running testDeleteKey");
String keyName = "key" + RandomStringUtils.randomNumeric(5);
OzoneBucket bucket = creatBucket();
- String volumeName = bucket.getBucketInfo().getVolumeName();
- String bucketName = bucket.getBucketName();
- bucket.putKey(keyName, "test-data");
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getName();
+ String dataStr = "test-data";
+ OzoneOutputStream keyOutputStream =
+ bucket.createKey(keyName, dataStr.length());
+ keyOutputStream.write(dataStr.getBytes());
+ keyOutputStream.close();
- OzoneKey keyInfo = bucket.getKeyInfo(keyName);
- assertEquals(keyName, keyInfo.getObjectInfo().getKeyName());
+ OzoneKey keyInfo = bucket.getKey(keyName);
+ assertEquals(keyName, keyInfo.getName());
String[] args = new String[] {"-deleteKey",
url + "/" + volumeName + "/" + bucketName + "/" + keyName};
@@ -618,9 +713,9 @@ public class TestOzoneShell {
// verify if key has been deleted in the bucket
try {
- bucket.getKeyInfo(keyName);
+ bucket.getKey(keyName);
fail("Get key should have thrown.");
- } catch (OzoneException e) {
+ } catch (IOException e) {
GenericTestUtils.assertExceptionContains(
"Lookup key failed, error:KEY_NOT_FOUND", e);
}
@@ -643,22 +738,29 @@ public class TestOzoneShell {
@Test
public void testInfoKey() throws Exception {
+ LOG.info("Running testInfoKey");
String keyName = "key" + RandomStringUtils.randomNumeric(5);
OzoneBucket bucket = creatBucket();
- String volumeName = bucket.getBucketInfo().getVolumeName();
- String bucketName = bucket.getBucketName();
- bucket.putKey(keyName, "test-data");
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getName();
+ String dataStr = "test-data";
+ OzoneOutputStream keyOutputStream =
+ bucket.createKey(keyName, dataStr.length());
+ keyOutputStream.write(dataStr.getBytes());
+ keyOutputStream.close();
String[] args = new String[] {"-infoKey",
url + "/" + volumeName + "/" + bucketName + "/" + keyName};
// verify the response output
- assertEquals(0, ToolRunner.run(shell, args));
-
+ int a = ToolRunner.run(shell, args);
String output = out.toString();
+ assertEquals(0, a);
+
assertTrue(output.contains(keyName));
- assertTrue(output.contains("createdOn") && output.contains("modifiedOn")
- && output.contains(OzoneConsts.OZONE_TIME_ZONE));
+ assertTrue(
+ output.contains("createdOn") && output.contains("modifiedOn") && output
+ .contains(OzoneConsts.OZONE_TIME_ZONE));
// reset stream
out.reset();
@@ -677,19 +779,27 @@ public class TestOzoneShell {
@Test
public void testListKey() throws Exception {
+ LOG.info("Running testListKey");
+ if (clientProtocol.equals(RestClient.class)) {
+ return;
+ }
String commandOutput;
List<KeyInfo> keys;
int keyCount = 11;
OzoneBucket bucket = creatBucket();
- String volumeName = bucket.getBucketInfo().getVolumeName();
- String bucketName = bucket.getBucketName();
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getName();
String keyName;
List<String> keyNames = new ArrayList<>();
for (int i = 0; i < keyCount; i++) {
keyName = "test-key" + i;
keyNames.add(keyName);
- bucket.putKey(keyName, "test-data" + i);
+ String dataStr = "test-data";
+ OzoneOutputStream keyOutputStream =
+ bucket.createKey(keyName, dataStr.length());
+ keyOutputStream.write(dataStr.getBytes());
+ keyOutputStream.close();
}
// test -length option
@@ -763,17 +873,23 @@ public class TestOzoneShell {
"the vaule should be a positive number"));
}
- private OzoneVolume creatVolume() throws OzoneException {
- String volumeName = UUID.randomUUID().toString() + "volume";
- OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
+ private OzoneVolume creatVolume() throws OzoneException, IOException {
+ String volumeName = RandomStringUtils.randomNumeric(5) + "volume";
+ VolumeArgs volumeArgs = VolumeArgs.newBuilder()
+ .setOwner("bilbo")
+ .setQuota("100TB")
+ .build();
+ client.createVolume(volumeName, volumeArgs);
+ OzoneVolume volume = client.getVolumeDetails(volumeName);
- return vol;
+ return volume;
}
- private OzoneBucket creatBucket() throws OzoneException {
+ private OzoneBucket creatBucket() throws OzoneException, IOException {
OzoneVolume vol = creatVolume();
- String bucketName = UUID.randomUUID().toString() + "bucket";
- OzoneBucket bucketInfo = vol.createBucket(bucketName);
+ String bucketName = RandomStringUtils.randomNumeric(5) + "bucket";
+ vol.createBucket(bucketName);
+ OzoneBucket bucketInfo = vol.getBucket(bucketName);
return bucketInfo;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
index 46539e7..64e5f71 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
@@ -132,7 +133,7 @@ public class TestBuckets {
fail("Except the bucket creation to be failed because the"
+ " bucket name starts with an invalid char #");
} catch (Exception e) {
- assertTrue(e instanceof OzoneRestClientException);
+ assertTrue(e instanceof OzoneClientException);
assertTrue(e.getMessage().contains("Bucket or Volume name"
+ " has an unsupported character : #"));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
index 4cd90c9..2d3cea9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.Status;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.ozone.client.rest.OzoneException;
@@ -141,7 +142,7 @@ public class TestVolume {
fail("Except the volume creation be failed because the"
+ " volume name starts with an invalid char #");
} catch (Exception e) {
- assertTrue(e instanceof OzoneRestClientException);
+ assertTrue(e instanceof OzoneClientException);
assertTrue(e.getMessage().contains("Bucket or Volume name"
+ " has an unsupported character : #"));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
index 99020c8..7fe6bb8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
@@ -19,28 +19,30 @@
package org.apache.hadoop.ozone.web.ozShell;
import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClient;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_HTTP_SCHEME;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
+
/**
* Common interface for command handling.
*/
public abstract class Handler {
- protected OzoneRestClient client;
-
- /**
- * Constructs a client object.
- */
- public Handler() {
- client = new OzoneRestClient();
- }
+ protected static final Logger LOG = LoggerFactory.getLogger(Handler.class);
+ protected OzoneClient client;
/**
* Executes the Client command.
@@ -61,19 +63,44 @@ public abstract class Handler {
* @throws URISyntaxException
* @throws OzoneException
*/
- protected URI verifyURI(String uri) throws URISyntaxException,
- OzoneException {
+ protected URI verifyURI(String uri)
+ throws URISyntaxException, OzoneException, IOException {
if ((uri == null) || uri.isEmpty()) {
- throw new OzoneRestClientException(
+ throw new OzoneClientException(
"Ozone URI is needed to execute this command.");
}
URIBuilder ozoneURI = new URIBuilder(uri);
-
if (ozoneURI.getPort() == 0) {
ozoneURI.setPort(Shell.DEFAULT_OZONE_PORT);
}
+
+ Configuration conf = new OzoneConfiguration();
+ String scheme = ozoneURI.getScheme();
+ if (scheme.equals(OZONE_HTTP_SCHEME)) {
+ if (ozoneURI.getHost() != null) {
+ if (ozoneURI.getPort() == -1) {
+ client = OzoneClientFactory.getRestClient(ozoneURI.getHost());
+ } else {
+ client = OzoneClientFactory
+ .getRestClient(ozoneURI.getHost(), ozoneURI.getPort(), conf);
+ }
+ } else {
+ client = OzoneClientFactory.getRestClient(conf);
+ }
+ } else if (scheme.equals(OZONE_URI_SCHEME) || scheme.isEmpty()) {
+ if (ozoneURI.getHost() != null) {
+ if (ozoneURI.getPort() == -1) {
+ client = OzoneClientFactory.getRpcClient(ozoneURI.getHost());
+ } else {
+ client = OzoneClientFactory
+ .getRpcClient(ozoneURI.getHost(), ozoneURI.getPort(), conf);
+ }
+ } else {
+ client = OzoneClientFactory.getRpcClient(conf);
+ }
+ } else {
+ throw new OzoneClientException("Invalid URI: " + ozoneURI);
+ }
return ozoneURI.build();
}
-
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db209ba/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
index d1c46b5..0788f9e 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
@@ -18,9 +18,10 @@
package org.apache.hadoop.ozone.web.ozShell.bucket;
import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientUtils;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.web.ozShell.Handler;
import org.apache.hadoop.ozone.web.ozShell.Shell;
@@ -39,7 +40,6 @@ public class CreateBucketHandler extends Handler {
private String volumeName;
private String bucketName;
- private String rootName;
/**
* Executes create bucket.
@@ -54,7 +54,7 @@ public class CreateBucketHandler extends Handler {
protected void execute(CommandLine cmd)
throws IOException, OzoneException, URISyntaxException {
if (!cmd.hasOption(Shell.CREATE_BUCKET)) {
- throw new OzoneRestClientException(
+ throw new OzoneClientException(
"Incorrect call : createBucket is missing");
}
@@ -62,7 +62,7 @@ public class CreateBucketHandler extends Handler {
URI ozoneURI = verifyURI(ozoneURIString);
Path path = Paths.get(ozoneURI.getPath());
if (path.getNameCount() < 2) {
- throw new OzoneRestClientException(
+ throw new OzoneClientException(
"volume and bucket name required in createBucket");
}
@@ -74,23 +74,13 @@ public class CreateBucketHandler extends Handler {
System.out.printf("Bucket Name : %s%n", bucketName);
}
- if (cmd.hasOption(Shell.RUNAS)) {
- rootName = "hdfs";
- } else {
- rootName = System.getProperty("user.name");
- }
-
-
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(rootName);
-
-
- OzoneVolume vol = client.getVolume(volumeName);
- OzoneBucket bucket = vol.createBucket(bucketName);
+ OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+ vol.createBucket(bucketName);
if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
- bucket.getBucketInfo().toJsonString()));
+ OzoneBucket bucket = vol.getBucket(bucketName);
+ System.out.printf(JsonUtils.toJsonStringWithDefaultPrettyPrinter(
+ JsonUtils.toJsonString(OzoneClientUtils.asBucketInfo(bucket))));
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org