You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2022/11/18 02:57:53 UTC
[ozone] branch HDDS-6517-Snapshot updated: HDDS-6854. [Snapshot] Implement List Snapshot API (#3784)
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-6517-Snapshot
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-6517-Snapshot by this push:
new 67e67250bd HDDS-6854. [Snapshot] Implement List Snapshot API (#3784)
67e67250bd is described below
commit 67e67250bd000f96654dda24d94262e6e5a8907b
Author: Chung En Lee <wf...@gmail.com>
AuthorDate: Fri Nov 18 10:57:47 2022 +0800
HDDS-6854. [Snapshot] Implement List Snapshot API (#3784)
---
.../apache/hadoop/ozone/client/ObjectStore.java | 12 +-
.../apache/hadoop/ozone/client/OzoneSnapshot.java | 145 +++++++++++++++++++++
.../ozone/client/protocol/ClientProtocol.java | 12 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 20 +++
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 +
.../org/apache/hadoop/ozone/audit/OMAction.java | 3 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 14 ++
...OzoneManagerProtocolClientSideTranslatorPB.java | 21 +++
.../src/main/proto/OmClientProtocol.proto | 13 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 9 ++
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 17 +++
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 61 +++++++++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 28 ++++
.../protocolPB/OzoneManagerRequestHandler.java | 18 +++
.../hadoop/ozone/om/TestOmMetadataManager.java | 39 ++++++
.../ozone/om/request/OMRequestTestUtils.java | 39 ++++++
.../hadoop/ozone/client/ClientProtocolStub.java | 6 +
.../ozone/shell/snapshot/ListSnapshotHandler.java | 61 +++++++++
.../ozone/shell/snapshot/SnapshotCommands.java | 1 +
19 files changed, 516 insertions(+), 4 deletions(-)
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index 55243ed31b..f500763697 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -549,5 +549,15 @@ public class ObjectStore {
return proxy.createSnapshot(volumeName, bucketName, snapshotName);
}
-
+ /**
+ * List snapshots in a volume/bucket.
+ * @param volumeName volume name
+ * @param bucketName bucket name
+ * @return list of snapshots for volume/bucket snapshotpath.
+ * @throws IOException
+ */
+ public List<OzoneSnapshot> listSnapshot(String volumeName, String bucketName)
+ throws IOException {
+ return proxy.listSnapshot(volumeName, bucketName);
+ }
}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshot.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshot.java
new file mode 100644
index 0000000000..6ddb5854e9
--- /dev/null
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshot.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.client;
+
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus;
+
+/**
+ * A class that encapsulates OzoneSnapshot.
+ */
+public class OzoneSnapshot {
+
+ private final String volumeName;
+ private final String bucketName;
+ private final String name;
+ private final long creationTime;
+ private final SnapshotStatus snapshotStatus;
+ private final String snapshotID; // UUID
+ private final String snapshotPath; // snapshot mask
+ private final String checkpointDir;
+
+ /**
+ * Constructs OzoneSnapshot from SnapshotInfo.
+ * @param volumeName Name of the Volume the snapshot belongs to.
+ * @param bucketName Name of the Bucket the snapshot belongs to.
+ * @param name Name of the snapshot.
+ * @param creationTime Creation time of the snapshot.
+ * @param snapshotStatus Status of the snapshot.
+ * @param snapshotID ID of the snapshot.
+ * @param snapshotPath Path of the snapshot.
+ * @param checkpointDir Snapshot checkpoint directory.
+ */
+ @SuppressWarnings("parameternumber")
+ public OzoneSnapshot(String volumeName, String bucketName,
+ String name, long creationTime,
+ SnapshotStatus snapshotStatus,
+ String snapshotID, String snapshotPath,
+ String checkpointDir) {
+ this.volumeName = volumeName;
+ this.bucketName = bucketName;
+ this.name = name;
+ this.creationTime = creationTime;
+ this.snapshotStatus = snapshotStatus;
+ this.snapshotID = snapshotID;
+ this.snapshotPath = snapshotPath;
+ this.checkpointDir = checkpointDir;
+ }
+
+ /**
+ * Returns volume name associated with the snapshot.
+ *
+ * @return volumeName
+ */
+ public String getVolumeName() {
+ return volumeName;
+ }
+
+ /**
+ * Returns bucket name associated with the snapshot.
+ *
+ * @return bucketName
+ */
+ public String getBucketName() {
+ return bucketName;
+ }
+
+ /**
+ * Returns name associated with the snapshot.
+ *
+ * @return name
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Returns the creation time of the snapshot.
+ *
+ * @return creationTime
+ */
+ public long getCreationTime() {
+ return creationTime;
+ }
+
+ /**
+ * Returns the status of the snapshot.
+ *
+ * @return snapshotStatus
+ */
+ public String getSnapshotStatus() {
+ return snapshotStatus.name();
+ }
+
+ /**
+ * Returns ID of the snapshot.
+ *
+ * @return snapshotID
+ */
+ public String getSnapshotID() {
+ return snapshotID;
+ }
+
+ /**
+ * Returns path of the snapshot.
+ *
+ * @return snapshotPath
+ */
+ public String getSnapshotPath() {
+ return snapshotPath;
+ }
+
+ /**
+ * Return snapshot checkpoint directory.
+ *
+ * @return snapshotCheckpointDir
+ */
+ public String getCheckpointDir() {
+ return checkpointDir;
+ }
+ public static OzoneSnapshot fromSnapshotInfo(SnapshotInfo snapshotInfo) {
+ return new OzoneSnapshot(
+ snapshotInfo.getVolumeName(),
+ snapshotInfo.getBucketName(),
+ snapshotInfo.getName(),
+ snapshotInfo.getCreationTime(),
+ snapshotInfo.getSnapshotStatus(),
+ snapshotInfo.getSnapshotID(),
+ snapshotInfo.getSnapshotPath(),
+ snapshotInfo.getCheckpointDir());
+ }
+}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 392004c618..1c092a4708 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.OzoneMultipartUploadList;
import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
+import org.apache.hadoop.ozone.client.OzoneSnapshot;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.TenantArgs;
import org.apache.hadoop.ozone.client.VolumeArgs;
@@ -968,5 +969,14 @@ public interface ClientProtocol {
*/
String createSnapshot(String volumeName,
String bucketName, String snapshotName) throws IOException;
-
+
+ /**
+ * List snapshots in a volume/bucket.
+ * @param volumeName volume name
+ * @param bucketName bucket name
+ * @return list of snapshots for volume/bucket snapshotpath.
+ * @throws IOException
+ */
+ List<OzoneSnapshot> listSnapshot(String volumeName, String bucketName)
+ throws IOException;
}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 0565071036..ef5e377788 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -88,6 +88,7 @@ import org.apache.hadoop.ozone.client.OzoneKeyLocation;
import org.apache.hadoop.ozone.client.OzoneMultipartUpload;
import org.apache.hadoop.ozone.client.OzoneMultipartUploadList;
import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
+import org.apache.hadoop.ozone.client.OzoneSnapshot;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.TenantArgs;
import org.apache.hadoop.ozone.client.VolumeArgs;
@@ -931,6 +932,25 @@ public class RpcClient implements ClientProtocol {
bucketName, snapshotName);
}
+ /**
+ * List snapshots in a volume/bucket.
+ * @param volumeName volume name
+ * @param bucketName bucket name
+ * @return list of snapshots for volume/bucket snapshotpath.
+ * @throws IOException
+ */
+ @Override
+ public List<OzoneSnapshot> listSnapshot(String volumeName, String bucketName)
+ throws IOException {
+ Preconditions.checkArgument(Strings.isNotBlank(volumeName),
+ "volume can't be null or empty.");
+ Preconditions.checkArgument(Strings.isNotBlank(bucketName),
+ "bucket can't be null or empty.");
+ return ozoneManagerClient.listSnapshot(volumeName, bucketName).stream()
+ .map(snapshotInfo -> OzoneSnapshot.fromSnapshotInfo(snapshotInfo))
+ .collect(Collectors.toList());
+ }
+
/**
* Assign admin role to an accessId in a tenant.
* @param accessId access ID.
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index ab7bacb05b..72eef66157 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -272,6 +272,7 @@ public final class OmUtils {
case ListTenant:
case TenantGetUserInfo:
case TenantListUser:
+ case ListSnapshot:
case EchoRPC:
case RangerBGSync:
// RangerBGSync is a read operation in the sense that it doesn't directly
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
index 13d8b16598..c691b18ac0 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
@@ -89,7 +89,8 @@ public enum OMAction implements AuditAction {
TENANT_ASSIGN_ADMIN,
TENANT_REVOKE_ADMIN,
TENANT_LIST_USER,
- CREATE_SNAPSHOT;
+ CREATE_SNAPSHOT,
+ LIST_SNAPSHOT;
@Override
public String getAction() {
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index d4b0ffe209..aef8d31f4d 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.helpers.TenantStateList;
import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue;
import org.apache.hadoop.ozone.om.helpers.TenantUserList;
@@ -621,6 +622,19 @@ public interface OzoneManagerProtocol
"this to be implemented");
}
+ /**
+ * List snapshots in a volume/bucket.
+ * @param volumeName volume name
+ * @param bucketName bucket name
+ * @return list of snapshots for volume/bucket snapshotpath.
+ * @throws IOException
+ */
+ default List<SnapshotInfo> listSnapshot(String volumeName, String bucketName)
+ throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented");
+ }
+
/**
* Assign admin role to a user identified by an accessId in a tenant.
* @param accessId access ID.
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 57595029a9..09439586fc 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -1116,6 +1116,27 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
return snapshotInfo.getName();
}
+ @Override
+ public List<SnapshotInfo> listSnapshot(String volumeName, String bucketName)
+ throws IOException {
+ final OzoneManagerProtocolProtos.ListSnapshotRequest.Builder
+ requestBuilder =
+ OzoneManagerProtocolProtos.ListSnapshotRequest.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName);
+
+ final OMRequest omRequest = createOMRequest(Type.ListSnapshot)
+ .setListSnapshotRequest(requestBuilder)
+ .build();
+ final OMResponse omResponse = submitRequest(omRequest);
+ handleError(omResponse);
+ List<SnapshotInfo> snapshotInfos = omResponse.getListSnapshotResponse()
+ .getSnapshotInfoList().stream()
+ .map(snapshotInfo -> SnapshotInfo.getFromProtobuf(snapshotInfo))
+ .collect(Collectors.toList());
+ return snapshotInfos;
+ }
+
/**
* {@inheritDoc}
*/
diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 27787143f7..9380d35068 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -126,6 +126,7 @@ enum Type {
EchoRPC = 110;
CreateSnapshot = 111;
+ ListSnapshot = 112;
}
message OMRequest {
@@ -234,6 +235,7 @@ message OMRequest {
optional EchoRPCRequest EchoRPCRequest = 110;
optional CreateSnapshotRequest CreateSnapshotRequest = 111;
+ optional ListSnapshotRequest ListSnapshotRequest = 112;
}
@@ -336,7 +338,7 @@ message OMResponse {
optional EchoRPCResponse EchoRPCResponse = 110;
optional CreateSnapshotResponse CreateSnapshotResponse = 111;
-
+ optional ListSnapshotResponse ListSnapshotResponse = 112;
}
enum Status {
@@ -1643,6 +1645,11 @@ message CreateSnapshotRequest {
optional string snapshotName = 3;
}
+message ListSnapshotRequest {
+ required string volumeName = 1;
+ required string bucketName = 2;
+}
+
message DeleteTenantRequest {
optional string tenantId = 1;
}
@@ -1684,6 +1691,10 @@ message CreateSnapshotResponse {
required SnapshotInfo snapshotInfo = 1;
}
+message ListSnapshotResponse {
+ repeated SnapshotInfo snapshotInfo = 1;
+}
+
message DeleteTenantResponse {
optional string volumeName = 1;
optional int64 volRefCount = 2;
diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index fe5ab879d8..b4ac54f44a 100644
--- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -222,6 +222,15 @@ public interface OMMetadataManager extends DBStoreHAManager {
List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName,
String startKeyName, String keyPrefix, int maxKeys) throws IOException;
+ /**
+ * List snapshots in a volume/bucket.
+ * @param volumeName volume name
+ * @param bucketName bucket name
+ * @return list of snapshot
+ */
+ List<SnapshotInfo> listSnapshot(String volumeName, String bucketName)
+ throws IOException;
+
/**
* Recover trash allows the user to recover the keys
* that were marked as deleted, but not actually deleted by Ozone Manager.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
index d700b2a620..fbf1fb6ca7 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
@@ -69,6 +69,7 @@ public class OMMetrics implements OmMetadataReaderMetrics {
private @Metric MutableCounterLong numInitiateMultipartUploads;
private @Metric MutableCounterLong numCompleteMultipartUploads;
private @Metric MutableCounterLong numSnapshotCreates;
+ private @Metric MutableCounterLong numSnapshotLists;
private @Metric MutableCounterLong numGetFileStatus;
private @Metric MutableCounterLong numCreateDirectory;
@@ -117,6 +118,7 @@ public class OMMetrics implements OmMetadataReaderMetrics {
private @Metric MutableCounterLong numListMultipartUploadPartFails;
private @Metric MutableCounterLong numOpenKeyDeleteRequestFails;
private @Metric MutableCounterLong numSnapshotCreateFails;
+ private @Metric MutableCounterLong numSnapshotListFails;
// Number of tenant operations attempted
private @Metric MutableCounterLong numTenantOps;
@@ -430,6 +432,13 @@ public class OMMetrics implements OmMetadataReaderMetrics {
numSnapshotCreateFails.incr();
}
+ public void incNumSnapshotLists() {
+ numSnapshotLists.incr();
+ }
+
+ public void incNumSnapshotListFails() {
+ numSnapshotListFails.incr();
+ }
public void incNumCompleteMultipartUploadFails() {
numCompleteMultipartUploadFails.incr();
@@ -1103,10 +1112,18 @@ public class OMMetrics implements OmMetadataReaderMetrics {
return numSnapshotCreates.value();
}
+ public long getNumSnapshotLists() {
+ return numSnapshotLists.value();
+ }
+
public long getNumSnapshotCreateFails() {
return numSnapshotCreateFails.value();
}
+ public long getNumSnapshotListFails() {
+ return numSnapshotListFails.value();
+ }
+
public void incNumTrashRenames() {
numTrashRenames.incr();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index e78df5a4a5..295821b0d9 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -1165,6 +1165,67 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
return deletedKeys;
}
+ @Override
+ public List<SnapshotInfo> listSnapshot(String volumeName, String bucketName)
+ throws IOException {
+ if (Strings.isNullOrEmpty(volumeName)) {
+ throw new OMException("Volume name is required.",
+ ResultCodes.VOLUME_NOT_FOUND);
+ }
+
+ if (Strings.isNullOrEmpty(bucketName)) {
+ throw new OMException("Bucket name is required.",
+ ResultCodes.BUCKET_NOT_FOUND);
+ }
+
+ String prefix = getBucketKey(volumeName, bucketName + OM_KEY_PREFIX);
+ TreeMap<String, SnapshotInfo> snapshotInfoMap = new TreeMap<>();
+
+ appendSnapshotFromCacheToMap(snapshotInfoMap, prefix);
+ appendSnapshotFromDBToMap(snapshotInfoMap, prefix);
+
+ return new ArrayList<>(snapshotInfoMap.values());
+ }
+
+ private void appendSnapshotFromCacheToMap(
+ TreeMap snapshotInfoMap, String prefix) {
+ Iterator<Map.Entry<CacheKey<String>, CacheValue<SnapshotInfo>>> iterator =
+ snapshotInfoTable.cacheIterator();
+ while (iterator.hasNext()) {
+ Map.Entry<CacheKey<String>, CacheValue<SnapshotInfo>> entry =
+ iterator.next();
+ String snapshotKey = entry.getKey().getCacheKey();
+ SnapshotInfo snapshotInfo = entry.getValue().getCacheValue();
+ if (snapshotInfo != null && snapshotKey.startsWith(prefix)) {
+ snapshotInfoMap.put(snapshotKey, snapshotInfo);
+ }
+ }
+ }
+
+ private void appendSnapshotFromDBToMap(TreeMap snapshotInfoMap, String prefix)
+ throws IOException {
+ try (TableIterator<String, ? extends KeyValue<String, SnapshotInfo>>
+ snapshotIter = snapshotInfoTable.iterator()) {
+ KeyValue< String, SnapshotInfo> snapshotinfo;
+ while (snapshotIter.hasNext()) {
+ snapshotinfo = snapshotIter.next();
+ if (snapshotinfo != null && snapshotinfo.getKey().startsWith(prefix)) {
+ CacheValue<SnapshotInfo> cacheValue =
+ snapshotInfoTable.getCacheValue(
+ new CacheKey<>(snapshotinfo.getKey()));
+ // There is always the latest data in the cache, so don't need to add
+ // earlier data from DB. We only add data from DB if there is no data
+ // in cache.
+ if (cacheValue == null) {
+ snapshotInfoMap.put(snapshotinfo.getKey(), snapshotinfo.getValue());
+ }
+ } else {
+ break;
+ }
+ }
+ }
+ }
+
@Override
public boolean recoverTrash(String volumeName, String bucketName,
String keyName, String destinationBucket) throws IOException {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 8013e1bd15..eb77df3782 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.OzoneManagerVersion;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.multitenant.OMRangerBGSyncService;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.util.OzoneNetUtils;
@@ -2787,6 +2788,33 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
}
}
+ @Override
+ public List<SnapshotInfo> listSnapshot(String volumeName, String bucketName)
+ throws IOException {
+ if (isAclEnabled) {
+ omMetadataReader.checkAcls(ResourceType.BUCKET, StoreType.OZONE,
+ ACLType.LIST, volumeName, bucketName, null);
+ }
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = buildAuditMap(volumeName);
+ auditMap.put(OzoneConsts.BUCKET, bucketName);
+ try {
+ metrics.incNumSnapshotLists();
+ return metadataManager.listSnapshot(volumeName, bucketName);
+ } catch (Exception ex) {
+ metrics.incNumSnapshotListFails();
+ auditSuccess = false;
+ AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_SNAPSHOT,
+ auditMap, ex));
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ AUDIT.logReadSuccess(buildAuditMessageForSuccess(
+ OMAction.LIST_SNAPSHOT, auditMap));
+ }
+ }
+ }
+
private Map<String, String> buildAuditMap(String volume) {
Map<String, String> auditMap = new LinkedHashMap<>();
auditMap.put(OzoneConsts.VOLUME, volume);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 7f20d16640..4f8b1e5528 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.helpers.TenantStateList;
import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue;
import org.apache.hadoop.ozone.om.helpers.TenantUserList;
@@ -277,6 +278,11 @@ public class OzoneManagerRequestHandler implements RequestHandler {
request.getTenantListUserRequest());
responseBuilder.setTenantListUserResponse(listUserResponse);
break;
+ case ListSnapshot:
+ OzoneManagerProtocolProtos.ListSnapshotResponse listSnapshotResponse =
+ getSnapshots(request.getListSnapshotRequest());
+ responseBuilder.setListSnapshotResponse(listSnapshotResponse);
+ break;
default:
responseBuilder.setSuccess(false);
responseBuilder.setMessage("Unrecognized Command Type: " + cmdType);
@@ -1195,4 +1201,16 @@ public class OzoneManagerRequestHandler implements RequestHandler {
public OzoneManager getOzoneManager() {
return impl;
}
+
+ private OzoneManagerProtocolProtos.ListSnapshotResponse getSnapshots(
+ OzoneManagerProtocolProtos.ListSnapshotRequest request)
+ throws IOException {
+ List<SnapshotInfo> snapshotInfos = impl.listSnapshot(
+ request.getVolumeName(), request.getBucketName());
+ List<OzoneManagerProtocolProtos.SnapshotInfo> snapshotInfoList =
+ snapshotInfos.stream().map(SnapshotInfo::getProtobuf)
+ .collect(Collectors.toList());
+ return OzoneManagerProtocolProtos.ListSnapshotResponse.newBuilder()
+ .addAllSnapshotInfo(snapshotInfoList).build();
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
index 6291f72620..5553ce2e92 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
@@ -22,12 +22,14 @@ import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKey;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket;
@@ -666,4 +668,41 @@ public class TestOmMetadataManager {
Assert.assertEquals(tablesByDefinition, tablesInManager);
}
+
+ @Test
+ public void testListSnapshot() throws Exception {
+ String vol1 = "vol1";
+ String bucket1 = "bucket1";
+
+ OMRequestTestUtils.addVolumeToDB(vol1, omMetadataManager);
+ addBucketsToCache(vol1, bucket1);
+ String snapshotName = "snapshot";
+
+ for (int i = 1; i <= 10; i++) {
+ if (i % 2 == 0) {
+ OMRequestTestUtils.addSnapshotToTable(vol1, bucket1,
+ snapshotName + i, omMetadataManager);
+ } else {
+ OMRequestTestUtils.addSnapshotToTableCache(vol1, bucket1,
+ snapshotName + i, omMetadataManager);
+ }
+ }
+
+ //Test listing snapshots with no volume name.
+ Assert.assertThrows(OMException.class, () -> omMetadataManager.listSnapshot(
+ null, null));
+
+ //Test listing snapshots with no bucket name.
+ Assert.assertThrows(OMException.class, () -> omMetadataManager.listSnapshot(
+ vol1, null));
+
+ //Test listing all snapshots.
+ List<SnapshotInfo> snapshotInfos = omMetadataManager.listSnapshot(vol1,
+ bucket1);
+ Assert.assertEquals(10, snapshotInfos.size());
+ for (SnapshotInfo snapshotInfo : snapshotInfos) {
+ Assert.assertTrue(snapshotInfo.getName().startsWith(snapshotName));
+ }
+
+ }
}
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java
index 970ebe26b4..c6c2c4a7df 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateTenantRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteTenantRequest;
@@ -336,6 +337,44 @@ public final class OMRequestTestUtils {
omMetadataManager.getDirectoryTable().put(ozoneKey, omDirInfo);
}
+ /**
+ * Add snapshot entry to DB.
+ */
+ public static void addSnapshotToTable(
+ String volumeName, String bucketName, String snapshotName,
+ OMMetadataManager omMetadataManager) throws IOException {
+ SnapshotInfo snapshotInfo = SnapshotInfo.newInstance(volumeName,
+ bucketName, snapshotName);
+ addSnapshotToTable(false, 0L, snapshotInfo, omMetadataManager);
+ }
+
+ /**
+ * Add snapshot entry to snapshot table cache.
+ */
+ public static void addSnapshotToTableCache(
+ String volumeName, String bucketName, String snapshotName,
+ OMMetadataManager omMetadataManager) throws IOException {
+ SnapshotInfo snapshotInfo = SnapshotInfo.newInstance(volumeName, bucketName,
+ snapshotName);
+ addSnapshotToTable(true, 0L, snapshotInfo, omMetadataManager);
+ }
+
+ /**
+ * Add snapshot entry to snapshotInfoTable. If addToCache flag set true,
+ * add it to cache table, else add it to DB.
+ */
+ public static void addSnapshotToTable(
+ Boolean addToCache, long txnID, SnapshotInfo snapshotInfo,
+ OMMetadataManager omMetadataManager) throws IOException {
+ String key = snapshotInfo.getTableKey();
+ if (addToCache) {
+ omMetadataManager.getSnapshotInfoTable().addCacheEntry(
+ new CacheKey<>(key),
+ new CacheValue<>(Optional.of(snapshotInfo), txnID));
+ }
+ omMetadataManager.getSnapshotInfoTable().put(key, snapshotInfo);
+ }
+
/**
* Create OmKeyInfo.
*/
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
index 3cab2072e2..97c1338a9a 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
@@ -577,4 +577,10 @@ public class ClientProtocolStub implements ClientProtocol {
throws IOException {
return "";
}
+
+ @Override
+ public List<OzoneSnapshot> listSnapshot(String volumeName, String bucketName)
+ throws IOException {
+ return null;
+ }
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java
new file mode 100644
index 0000000000..e3440d04e5
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.snapshot;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneSnapshot;
+import org.apache.hadoop.ozone.shell.Handler;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.bucket.BucketUri;
+import picocli.CommandLine;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * a handler for Ozone shell CLI command 'list snapshot'.
+ */
+@CommandLine.Command(name = "list",
+ aliases = "ls",
+ description = "list snapshot for the buckets.")
+public class ListSnapshotHandler extends Handler {
+
+ @CommandLine.Mixin
+ private BucketUri snapshotPath;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return snapshotPath.getValue();
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException {
+ String volumeName = snapshotPath.getValue().getVolumeName();
+ String bucketName = snapshotPath.getValue().getBucketName();
+
+ List<? extends OzoneSnapshot> snapshotInfos = client.getObjectStore()
+ .listSnapshot(volumeName, bucketName);
+ int counter = printAsJsonArray(snapshotInfos.iterator(),
+ snapshotInfos.size());
+ if (isVerbose()) {
+ out().printf("Found : %d snapshots for o3://%s/ %s ", counter,
+ volumeName, bucketName);
+ }
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotCommands.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotCommands.java
index fdb3eff2ba..22a4de0414 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotCommands.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotCommands.java
@@ -39,6 +39,7 @@ import picocli.CommandLine.ParentCommand;
description = "Snapshot specific operations",
subcommands = {
CreateSnapshotHandler.class,
+ ListSnapshotHandler.class,
},
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org