You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by di...@apache.org on 2019/11/11 01:31:17 UTC
[hadoop-ozone] branch master updated: HDDS-2417 Add the list trash
command to the client side (#138)
This is an automated email from the ASF dual-hosted git repository.
dineshc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new f7ba616 HDDS-2417 Add the list trash command to the client side (#138)
f7ba616 is described below
commit f7ba61684460dbeff67886c6b7cbfcda7688ec2c
Author: Matthew Sharp <56...@users.noreply.github.com>
AuthorDate: Sun Nov 10 20:31:09 2019 -0500
HDDS-2417 Add the list trash command to the client side (#138)
---
.../ozone/client/protocol/ClientProtocol.java | 20 +++++++++++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 12 +++++++
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 19 ++++++++++
...OzoneManagerProtocolClientSideTranslatorPB.java | 42 ++++++++++++++++++++++
.../src/main/proto/OzoneManagerProtocol.proto | 21 +++++++++++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 11 ++++++
7 files changed, 126 insertions(+)
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 068c597..77345d9 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -41,6 +41,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
@@ -314,6 +315,25 @@ public interface ClientProtocol {
String keyPrefix, String prevKey, int maxListResult)
throws IOException;
+ /**
+ * List trash allows the user to list the keys that were marked as deleted,
+ * but not actually deleted by Ozone Manager. This allows a user to recover
+ * keys within a configurable window.
+ * @param volumeName - The volume name, which can also be a wild card
+ * using '*'.
+ * @param bucketName - The bucket name, which can also be a wild card
+ * using '*'.
+ * @param startKeyName - List keys from a specific key name.
+ * @param keyPrefix - List keys using a specific prefix.
+ * @param maxKeys - The number of keys to be returned. This must be below
+ * the cluster level set by admins.
+ * @return The list of keys that are deleted from the deleted table.
+ * @throws IOException
+ */
+ List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName,
+ String startKeyName, String keyPrefix,
+ int maxKeys)
+ throws IOException;
/**
* Get OzoneKey.
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index efc9470..302a561 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
@@ -709,6 +710,17 @@ public class RpcClient implements ClientProtocol {
}
@Override
+ public List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName,
+ String startKeyName, String keyPrefix, int maxKeys) throws IOException {
+
+ Preconditions.checkNotNull(volumeName);
+ Preconditions.checkNotNull(bucketName);
+
+ return ozoneManagerClient.listTrash(volumeName, bucketName, startKeyName,
+ keyPrefix, maxKeys);
+ }
+
+ @Override
public OzoneKeyDetails getKeyDetails(
String volumeName, String bucketName, String keyName)
throws IOException {
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 8e129c9..ca51187 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -224,6 +224,7 @@ public final class OmUtils {
case ListBuckets:
case LookupKey:
case ListKeys:
+ case ListTrash:
case InfoS3Bucket:
case ListS3Buckets:
case ServiceList:
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index 1cdc2fc..9bd5032 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
@@ -527,4 +528,22 @@ public interface OzoneManagerProtocol
OzoneManagerProtocolProtos.DBUpdatesRequest dbUpdatesRequest)
throws IOException;
+ /**
+ * List trash allows the user to list the keys that were marked as deleted,
+ * but not actually deleted by Ozone Manager. This allows a user to recover
+ * keys within a configurable window.
+ * @param volumeName - The volume name, which can also be a wild card
+ * using '*'.
+ * @param bucketName - The bucket name, which can also be a wild card
+ * using '*'.
+ * @param startKeyName - List keys from a specific key name.
+ * @param keyPrefix - List keys using a specific prefix.
+ * @param maxKeys - The number of keys to be returned. This must be below
+ * the cluster level set by admins.
+ * @return The list of keys that are deleted from the deleted table.
+ * @throws IOException
+ */
+ List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName,
+ String startKeyName, String keyPrefix, int maxKeys) throws IOException;
+
}
\ No newline at end of file
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 6924ee2..206bf15 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
@@ -105,6 +106,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListBuc
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListBucketsResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupKeyRequest;
@@ -1592,4 +1595,43 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
}
return statusList;
}
+
+ @Override
+ public List<RepeatedOmKeyInfo> listTrash(String volumeName,
+ String bucketName, String startKeyName, String keyPrefix, int maxKeys)
+ throws IOException {
+
+ Preconditions.checkArgument(Strings.isNullOrEmpty(volumeName),
+ "The volume name cannot be null or " +
+ "empty. Please enter a valid volume name or use '*' as a wild card");
+
+ Preconditions.checkArgument(Strings.isNullOrEmpty(bucketName),
+ "The bucket name cannot be null or " +
+ "empty. Please enter a valid bucket name or use '*' as a wild card");
+
+ ListTrashRequest trashRequest = ListTrashRequest.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setStartKeyName(startKeyName)
+ .setKeyPrefix(keyPrefix)
+ .setMaxKeys(maxKeys)
+ .build();
+
+ OMRequest omRequest = createOMRequest(Type.ListTrash)
+ .setListTrashRequest(trashRequest)
+ .build();
+
+ ListTrashResponse trashResponse =
+ handleError(submitRequest(omRequest)).getListTrashResponse();
+
+ List<RepeatedOmKeyInfo> deletedKeyList =
+ new ArrayList<>(trashResponse.getDeletedKeysCount());
+
+ deletedKeyList.addAll(
+ trashResponse.getDeletedKeysList().stream()
+ .map(RepeatedOmKeyInfo::getFromProto)
+ .collect(Collectors.toList()));
+
+ return deletedKeyList;
+ }
}
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index 569b4f1..62b7190 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -92,6 +92,8 @@ enum Type {
PurgeKeys = 81;
ListMultipartUploads = 82;
+
+ ListTrash = 91;
}
message OMRequest {
@@ -161,6 +163,8 @@ message OMRequest {
optional UpdateGetS3SecretRequest updateGetS3SecretRequest = 82;
optional ListMultipartUploadsRequest listMultipartUploadsRequest = 83;
+
+ optional ListTrashRequest listTrashRequest = 91;
}
message OMResponse {
@@ -230,6 +234,8 @@ message OMResponse {
optional PurgeKeysResponse purgeKeysResponse = 81;
optional ListMultipartUploadsResponse listMultipartUploadsResponse = 82;
+
+ optional ListTrashResponse listTrashResponse = 91;
}
enum Status {
@@ -302,6 +308,21 @@ enum Status {
INVALID_PART_ORDER = 56;
}
+/**
+ This command acts as a list command for deleted keys that are still present
+ in the deleted table on Ozone Manager.
+*/
+message ListTrashRequest {
+ required string volumeName = 1;
+ required string bucketName = 2;
+ optional string startKeyName = 3;
+ optional string keyPrefix = 4;
+ optional int32 maxKeys = 5;
+}
+
+message ListTrashResponse {
+ repeated RepeatedKeyInfo deletedKeys = 1;
+}
message VolumeInfo {
required string adminName = 1;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 0f9b58d..9061d1b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -78,6 +78,7 @@ import org.apache.hadoop.ozone.om.ha.OMFailoverProxyProvider;
import org.apache.hadoop.ozone.om.ha.OMHANodeDetails;
import org.apache.hadoop.ozone.om.ha.OMNodeDetails;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerServerProtocol;
import org.apache.hadoop.ozone.om.ratis.OMRatisSnapshotInfo;
@@ -2239,6 +2240,16 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
}
}
+ // TODO: HDDS-2418 - Complete stub below for server logic
+ @Override
+ public List<RepeatedOmKeyInfo> listTrash(String volumeName,
+ String bucketName, String startKeyName, String keyPrefix, int maxKeys)
+ throws IOException {
+
+ List<RepeatedOmKeyInfo> deletedKeys = new ArrayList<>();
+ return deletedKeys;
+ }
+
/**
* Sets bucket property from args.
*
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org