You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/10/19 20:18:44 UTC
[02/50] [abbrv] hadoop git commit: HDDS-683. Add a shell command to
provide ozone mapping for a S3Bucket. Contributed by Bharat Viswanadham.
HDDS-683. Add a shell command to provide ozone mapping for a S3Bucket. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c4364dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c4364dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c4364dd
Branch: refs/heads/HDDS-4
Commit: 4c4364ddd06927fa990d3e66d4eadbd4a7eb9474
Parents: 7347fa2
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Wed Oct 17 16:46:06 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Wed Oct 17 16:46:13 2018 -0700
----------------------------------------------------------------------
.../hadoop/ozone/ozShell/TestOzoneShell.java | 42 +++++++++
.../apache/hadoop/ozone/web/ozShell/Shell.java | 5 +-
.../web/ozShell/bucket/BucketCommands.java | 3 +-
.../web/ozShell/bucket/S3BucketMapping.java | 99 ++++++++++++++++++++
4 files changed, 147 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c4364dd/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
index 03efa1c..a7eecc0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
@@ -1124,6 +1124,48 @@ public class TestOzoneShell {
executeWithError(shell, args, "the length should be a positive number");
}
+ @Test
+ public void testS3BucketMapping() throws IOException {
+ String s3Bucket = "bucket1";
+ String commandOutput;
+ createS3Bucket("ozone", s3Bucket);
+ String volumeName = client.getOzoneVolumeName(s3Bucket);
+ String[] args = new String[] {"bucket", "path", url + "/" + s3Bucket};
+ if (url.startsWith("o3")) {
+ execute(shell, args);
+ commandOutput = out.toString();
+ assertTrue(commandOutput.contains("Volume name for S3Bucket is : " +
+ volumeName));
+ assertTrue(commandOutput.contains(OzoneConsts.OZONE_URI_SCHEME +"://" +
+ s3Bucket + "." + volumeName));
+ out.reset();
+ //Trying to get map for an unknown bucket
+ args = new String[] {"bucket", "path", url + "/" + "unknownbucket"};
+ executeWithError(shell, args, "S3_BUCKET_NOT_FOUND");
+ } else {
+ executeWithError(shell, args, "Ozone REST protocol does not support " +
+ "this operation");
+ }
+
+ // No bucket name
+ args = new String[] {"bucket", "path", url};
+ executeWithError(shell, args, "S3Bucket name is required");
+
+ // Invalid bucket name
+ args = new String[] {"bucket", "path", url + "/" + s3Bucket +
+ "/multipleslash"};
+ executeWithError(shell, args, "Invalid S3Bucket name. Delimiters (/) not" +
+ " allowed");
+ }
+
+ private void createS3Bucket(String userName, String s3Bucket) {
+ try {
+ client.createS3Bucket("ozone", s3Bucket);
+ } catch (IOException ex) {
+ GenericTestUtils.assertExceptionContains("S3_BUCKET_ALREADY_EXISTS", ex);
+ }
+ }
+
private OzoneVolume creatVolume() throws OzoneException, IOException {
String volumeName = RandomStringUtils.randomNumeric(5) + "volume";
VolumeArgs volumeArgs = VolumeArgs.newBuilder()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c4364dd/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
index 0f3969f..28ab900 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
@@ -51,7 +51,7 @@ public class Shell extends GenericCli {
public static final String OZONE_URI_DESCRIPTION = "Ozone URI could start "
+ "with o3:// or http(s):// or without prefix. REST protocol will "
+ "be used for http(s), RPC otherwise. URI may contain the host and port "
- + "of the SCM server. Both are optional. "
+ + "of the OM server. Both are optional. "
+ "If they are not specified it will be identified from "
+ "the config files.";
@@ -64,6 +64,9 @@ public class Shell extends GenericCli {
public static final String OZONE_KEY_URI_DESCRIPTION =
"URI of the volume/bucket/key.\n" + OZONE_URI_DESCRIPTION;
+ public static final String OZONE_S3BUCKET_URI_DESCRIPTION = "URI of the " +
+ "S3Bucket.\n" + OZONE_URI_DESCRIPTION;
+
// General options
public static final int DEFAULT_OZONE_PORT = 50070;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c4364dd/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java
index 870f4d7..617643c 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java
@@ -38,7 +38,8 @@ import picocli.CommandLine.ParentCommand;
ListBucketHandler.class,
CreateBucketHandler.class,
UpdateBucketHandler.class,
- DeleteBucketHandler.class
+ DeleteBucketHandler.class,
+ S3BucketMapping.class
},
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c4364dd/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/S3BucketMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/S3BucketMapping.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/S3BucketMapping.java
new file mode 100644
index 0000000..21ae746
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/S3BucketMapping.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.web.ozShell.bucket;
+
+import java.net.URI;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.web.ozShell.Handler;
+import org.apache.hadoop.ozone.web.ozShell.Shell;
+
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Parameters;
+
+/**
+ * S3Bucket mapping handler, which returns volume name and Ozone fs uri for
+ * that bucket.
+ */
+@Command(name = "path",
+ description = "Returns the ozone path for S3Bucket")
+public class S3BucketMapping extends Handler {
+
+ @Parameters(arity = "1..1", description = Shell
+ .OZONE_S3BUCKET_URI_DESCRIPTION)
+ private String uri;
+
+ /**
+ * Executes create bucket.
+ */
+ @Override
+ public Void call() throws Exception {
+
+ URI ozoneURI = verifyURI(uri);
+ Path path = Paths.get(ozoneURI.getPath());
+ int pathNameCount = path.getNameCount();
+ String errorMessage;
+
+ // When just uri is given as http://om:9874, we are getting pathCount
+ // still as 1, as getPath() is returning empty string.
+ // So for safer side check, whether it is an empty string
+ if (pathNameCount == 1) {
+ String s3Bucket = path.getName(0).toString();
+ if (StringUtils.isBlank(s3Bucket)) {
+ errorMessage = "S3Bucket name is required to get volume name and " +
+ "Ozone fs Uri";
+ throw new OzoneClientException(errorMessage);
+ }
+ }
+ if (pathNameCount != 1) {
+ if (pathNameCount < 1) {
+ errorMessage = "S3Bucket name is required to get volume name and " +
+ "Ozone fs Uri";
+ } else {
+ errorMessage = "Invalid S3Bucket name. Delimiters (/) not allowed in " +
+ "S3Bucket name";
+ }
+ throw new OzoneClientException(errorMessage);
+ }
+
+ String s3Bucket = path.getName(0).toString();
+ if (isVerbose()) {
+ System.out.printf("S3Bucket Name : %s%n", s3Bucket);
+ }
+
+ String mapping = client.getObjectStore().getOzoneBucketMapping(s3Bucket);
+ String volumeName = client.getObjectStore().getOzoneVolumeName(s3Bucket);
+
+ if (isVerbose()) {
+ System.out.printf("Mapping created for S3Bucket is : %s%n", mapping);
+ }
+
+ System.out.printf("Volume name for S3Bucket is : %s%n", volumeName);
+
+ String ozoneFsUri = String.format("%s://%s.%s", OzoneConsts
+ .OZONE_URI_SCHEME, s3Bucket, volumeName);
+
+ System.out.printf("Ozone FileSystem Uri is : %s%n", ozoneFsUri);
+
+ return null;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org