You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2023/02/03 04:30:29 UTC
[ozone] branch master updated: HDDS-7277. [Snapshot] Implement snapshot key listing in Ozone Shell (#4125)
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 3732f55c16 HDDS-7277. [Snapshot] Implement snapshot key listing in Ozone Shell (#4125)
3732f55c16 is described below
commit 3732f55c165c93ded140e25e6dc70e89488c099b
Author: Christos Bisias <ch...@gmail.com>
AuthorDate: Fri Feb 3 06:30:22 2023 +0200
HDDS-7277. [Snapshot] Implement snapshot key listing in Ozone Shell (#4125)
---
hadoop-ozone/tools/pom.xml | 5 +
.../apache/hadoop/ozone/shell/OzoneAddress.java | 36 +++++++
.../hadoop/ozone/shell/keys/ListKeyHandler.java | 39 ++++++--
.../shell/snapshot/BucketSnapshotHandler.java | 36 +++++++
.../hadoop/ozone/shell/snapshot/SnapshotUri.java | 47 +++++++++
.../hadoop/ozone/shell/TestOzoneAddress.java | 105 +++++++++++++--------
6 files changed, 222 insertions(+), 46 deletions(-)
diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml
index 2f92d0d087..7a9fafc20f 100644
--- a/hadoop-ozone/tools/pom.xml
+++ b/hadoop-ozone/tools/pom.xml
@@ -121,6 +121,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java
index af6d624ed7..6adf1cf701 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
import com.google.common.annotations.VisibleForTesting;
+
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_HTTP_SCHEME;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RPC_SCHEME;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY;
@@ -54,6 +55,8 @@ public class OzoneAddress {
private String bucketName = "";
+ private String snapshotNameWithIndicator = "";
+
private String keyName = "";
private boolean isPrefix = false;
@@ -293,6 +296,10 @@ public class OzoneAddress {
return bucketName;
}
+ public String getSnapshotNameWithIndicator() {
+ return snapshotNameWithIndicator;
+ }
+
public String getKeyName() {
return keyName;
}
@@ -343,6 +350,35 @@ public class OzoneAddress {
}
}
+ /**
+ * Checking for a volume and a bucket
+ * but also accepting a snapshot
+ * indicator and a snapshot name.
+ * If the keyName can't be considered
+ * a valid snapshot, an exception is thrown.
+ *
+ * @throws OzoneClientException
+ */
+ public void ensureSnapshotAddress()
+ throws OzoneClientException {
+ if (keyName.length() > 0) {
+ if (OmUtils.isBucketSnapshotIndicator(keyName)) {
+ snapshotNameWithIndicator = keyName;
+ } else {
+ throw new OzoneClientException(
+ "Delimiters (/) not allowed following " +
+ "a bucket name. Only a snapshot name with " +
+ "a snapshot indicator is accepted");
+ }
+ } else if (volumeName.length() == 0) {
+ throw new OzoneClientException(
+ "Volume name is missing.");
+ } else if (bucketName.length() == 0) {
+ throw new OzoneClientException(
+ "Bucket name is missing.");
+ }
+ }
+
public void ensureVolumeAddress() throws OzoneClientException {
if (keyName.length() != 0) {
throw new OzoneClientException(
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ListKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ListKeyHandler.java
index 741619bc21..7846b0b007 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ListKeyHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ListKeyHandler.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.ozone.shell.keys;
import java.io.IOException;
import java.util.Iterator;
+import com.google.common.base.Strings;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientException;
@@ -28,18 +29,18 @@ import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.shell.ListOptions;
import org.apache.hadoop.ozone.shell.OzoneAddress;
-import org.apache.hadoop.ozone.shell.bucket.BucketHandler;
+import org.apache.hadoop.ozone.shell.snapshot.BucketSnapshotHandler;
import picocli.CommandLine;
import picocli.CommandLine.Command;
/**
- * Executes List Keys.
+ * Executes List Keys for a bucket or snapshot.
*/
@Command(name = "list",
aliases = "ls",
- description = "list all keys in a given bucket")
-public class ListKeyHandler extends BucketHandler {
+ description = "list all keys in a given bucket or snapshot")
+public class ListKeyHandler extends BucketSnapshotHandler {
@CommandLine.Mixin
private ListOptions listOptions;
@@ -50,11 +51,25 @@ public class ListKeyHandler extends BucketHandler {
String volumeName = address.getVolumeName();
String bucketName = address.getBucketName();
+ String snapshotNameWithIndicator = address.getSnapshotNameWithIndicator();
+ String keyPrefix = "";
+
+ if (!Strings.isNullOrEmpty(snapshotNameWithIndicator)) {
+ keyPrefix += snapshotNameWithIndicator;
+
+ if (!Strings.isNullOrEmpty(listOptions.getPrefix())) {
+ keyPrefix += "/";
+ }
+ }
+
+ if (!Strings.isNullOrEmpty(listOptions.getPrefix())) {
+ keyPrefix += listOptions.getPrefix();
+ }
OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
OzoneBucket bucket = vol.getBucket(bucketName);
Iterator<? extends OzoneKey> keyIterator = bucket.listKeys(
- listOptions.getPrefix(), listOptions.getStartItem());
+ keyPrefix, listOptions.getStartItem());
int maxKeyLimit = listOptions.getLimit();
int counter = printAsJsonArray(keyIterator, maxKeyLimit);
@@ -64,8 +79,18 @@ public class ListKeyHandler extends BucketHandler {
out().println("Listing first " + maxKeyLimit + " entries of the " +
"result. Use --length (-l) to override max returned keys.");
} else if (isVerbose()) {
- out().printf("Found : %d keys for bucket %s in volume : %s ",
- counter, bucketName, volumeName);
+ if (!Strings.isNullOrEmpty(snapshotNameWithIndicator)) {
+ // snapshotValues[0] = ".snapshot"
+ // snapshotValues[1] = snapshot name
+ String[] snapshotValues = snapshotNameWithIndicator.split("/");
+
+ out().printf("Found : %d keys for snapshot %s " +
+ "under bucket %s in volume : %s ",
+ counter, snapshotValues[1], bucketName, volumeName);
+ } else {
+ out().printf("Found : %d keys for bucket %s in volume : %s ",
+ counter, bucketName, volumeName);
+ }
}
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/BucketSnapshotHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/BucketSnapshotHandler.java
new file mode 100644
index 0000000000..934e645d19
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/BucketSnapshotHandler.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.snapshot;
+
+import org.apache.hadoop.ozone.shell.Handler;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import picocli.CommandLine;
+
+/**
+ * Base class for bucket commands that require a snapshot URI.
+ */
+public abstract class BucketSnapshotHandler extends Handler {
+
+ @CommandLine.Mixin
+ private SnapshotUri address;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotUri.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotUri.java
new file mode 100644
index 0000000000..558a7a4bbd
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotUri.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.snapshot;
+
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.Shell;
+import picocli.CommandLine;
+
+/**
+ * URI parameter for snapshot-specific commands.
+ */
+public class SnapshotUri implements CommandLine.ITypeConverter<OzoneAddress> {
+
+ private static final String OZONE_SNAPSHOT_URI_DESCRIPTION =
+ "URI of the volume/bucket/snapshot.\n" + Shell.OZONE_URI_DESCRIPTION;
+
+ @CommandLine.Parameters(index = "0", arity = "1..1",
+ description = OZONE_SNAPSHOT_URI_DESCRIPTION,
+ converter = SnapshotUri.class)
+ private OzoneAddress value;
+
+ public OzoneAddress getValue() {
+ return value;
+ }
+
+ @Override
+ public OzoneAddress convert(String str) throws Exception {
+ OzoneAddress address = new OzoneAddress(str);
+ address.ensureSnapshotAddress();
+ return address;
+ }
+}
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneAddress.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneAddress.java
index ec651c6023..e831bf5f67 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneAddress.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneAddress.java
@@ -18,25 +18,21 @@
package org.apache.hadoop.ozone.shell;
-import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import org.apache.hadoop.ozone.client.OzoneClientException;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
/**
* Test ozone URL parsing.
*/
-@RunWith(Parameterized.class)
public class TestOzoneAddress {
- @Parameters
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
{"o3://localhost:9878/"},
@@ -47,63 +43,94 @@ public class TestOzoneAddress {
});
}
- private String prefix;
-
- public TestOzoneAddress(String prefix) {
- this.prefix = prefix;
- }
-
- @Test
- public void checkUrlTypes() throws OzoneClientException, IOException {
- OzoneAddress address;
+ private OzoneAddress address;
+ @ParameterizedTest
+ @MethodSource("data")
+ public void checkRootUrlType(String prefix) throws OzoneClientException {
address = new OzoneAddress("");
address.ensureRootAddress();
address = new OzoneAddress(prefix + "");
address.ensureRootAddress();
+ }
+ @ParameterizedTest
+ @MethodSource("data")
+ public void checkVolumeUrlType(String prefix) throws OzoneClientException {
address = new OzoneAddress(prefix + "vol1");
address.ensureVolumeAddress();
- Assert.assertEquals("vol1", address.getVolumeName());
+ Assertions.assertEquals("vol1", address.getVolumeName());
+ }
+ @ParameterizedTest
+ @MethodSource("data")
+ public void checkBucketUrlType(String prefix) throws OzoneClientException {
address = new OzoneAddress(prefix + "vol1/bucket");
address.ensureBucketAddress();
- Assert.assertEquals("vol1", address.getVolumeName());
- Assert.assertEquals("bucket", address.getBucketName());
+ Assertions.assertEquals("vol1", address.getVolumeName());
+ Assertions.assertEquals("bucket", address.getBucketName());
address = new OzoneAddress(prefix + "vol1/bucket/");
address.ensureBucketAddress();
- Assert.assertEquals("vol1", address.getVolumeName());
- Assert.assertEquals("bucket", address.getBucketName());
+ Assertions.assertEquals("vol1", address.getVolumeName());
+ Assertions.assertEquals("bucket", address.getBucketName());
+ }
+ @ParameterizedTest
+ @MethodSource("data")
+ public void checkKeyUrlType(String prefix) throws OzoneClientException {
address = new OzoneAddress(prefix + "vol1/bucket/key");
address.ensureKeyAddress();
- Assert.assertEquals("vol1", address.getVolumeName());
- Assert.assertEquals("bucket", address.getBucketName());
- Assert.assertEquals("key", address.getKeyName());
+ Assertions.assertEquals("vol1", address.getVolumeName());
+ Assertions.assertEquals("bucket", address.getBucketName());
+ Assertions.assertEquals("key", address.getKeyName());
address = new OzoneAddress(prefix + "vol1/bucket/key/");
address.ensureKeyAddress();
- Assert.assertEquals("vol1", address.getVolumeName());
- Assert.assertEquals("bucket", address.getBucketName());
- Assert.assertEquals("key/", address.getKeyName());
+ Assertions.assertEquals("vol1", address.getVolumeName());
+ Assertions.assertEquals("bucket", address.getBucketName());
+ Assertions.assertEquals("key/", address.getKeyName());
address = new OzoneAddress(prefix + "vol1/bucket/key1/key3/key");
address.ensureKeyAddress();
- Assert.assertEquals("vol1", address.getVolumeName());
- Assert.assertEquals("bucket", address.getBucketName());
- Assert.assertEquals("key1/key3/key", address.getKeyName());
- Assert.assertFalse("this should not be a prefix",
- address.isPrefix());
+ Assertions.assertEquals("vol1", address.getVolumeName());
+ Assertions.assertEquals("bucket", address.getBucketName());
+ Assertions.assertEquals("key1/key3/key", address.getKeyName());
+ Assertions.assertFalse(address.isPrefix(), "this should not be a prefix");
+ }
+ @ParameterizedTest
+ @MethodSource("data")
+ public void checkPrefixUrlType(String prefix) throws OzoneClientException {
address = new OzoneAddress(prefix + "vol1/bucket/prefix");
address.ensurePrefixAddress();
- Assert.assertEquals("vol1", address.getVolumeName());
- Assert.assertEquals("bucket", address.getBucketName());
- Assert.assertEquals("prefix", address.getKeyName());
- Assert.assertTrue("this should be a prefix",
- address.isPrefix());
+ Assertions.assertEquals("vol1", address.getVolumeName());
+ Assertions.assertEquals("bucket", address.getBucketName());
+ Assertions.assertEquals("prefix", address.getKeyName());
+ Assertions.assertTrue(address.isPrefix(), "this should be a prefix");
+ }
+
+ @ParameterizedTest
+ @MethodSource("data")
+ public void checkSnapshotUrlType(String prefix) throws OzoneClientException {
+ address = new OzoneAddress(prefix + "vol1/bucket/.snapshot/snap1");
+ address.ensureSnapshotAddress();
+ Assertions.assertEquals("vol1", address.getVolumeName());
+ Assertions.assertEquals("bucket", address.getBucketName());
+ Assertions.assertEquals(".snapshot/snap1",
+ address.getSnapshotNameWithIndicator());
+ Assertions.assertEquals(".snapshot/snap1", address.getKeyName());
+
+
+ String message = "Only a snapshot name with " +
+ "a snapshot indicator is accepted";
+
+ address = new OzoneAddress(prefix + "vol1/bucket/.snapshot");
+ OzoneClientException exception = Assertions
+ .assertThrows(OzoneClientException.class,
+ () -> address.ensureSnapshotAddress());
+ Assertions.assertTrue(exception.getMessage().contains(message));
}
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org