You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ra...@apache.org on 2021/04/07 03:25:44 UTC
[ozone] 17/29: HDDS-4742. Make trash work with FS Optimised
Buckets. (#1915)
This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a commit to branch HDDS-2939
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 4cb5091515a1fe6bb266da1ec701369932a28b5b
Author: Sadanand Shenoy <sa...@gmail.com>
AuthorDate: Tue Feb 16 22:07:12 2021 +0530
HDDS-4742. Make trash work with FS Optimised Buckets. (#1915)
---
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 25 ++++
.../hadoop/fs/ozone/TestOzoneFileSystemV1.java | 12 --
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 142 ++++++++++++++++-----
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 15 +--
4 files changed, 135 insertions(+), 59 deletions(-)
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
index e9d4cf9..e7e2eb0 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
@@ -18,10 +18,12 @@
package org.apache.hadoop.ozone.om.helpers;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.util.StringUtils;
import javax.annotation.Nonnull;
import java.nio.file.Paths;
+import java.util.Map;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
@@ -205,4 +207,27 @@ public final class OzoneFSUtils {
java.nio.file.Path keyPath = Paths.get(keyName);
return keyPath.getNameCount();
}
+
+
+ /**
+ * Returns true if the bucket is FS Optimised.
+ * @param bucketMetadata
+ * @return
+ */
+ public static boolean isFSOptimizedBucket(
+ Map<String, String> bucketMetadata) {
+ // layout version V1 represents optimized FS path
+ boolean layoutVersionEnabled =
+ org.apache.commons.lang3.StringUtils.equalsIgnoreCase(
+ OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1,
+ bucketMetadata
+ .get(OMConfigKeys.OZONE_OM_LAYOUT_VERSION));
+
+ boolean fsEnabled =
+ Boolean.parseBoolean(bucketMetadata
+ .get(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS));
+
+ return layoutVersionEnabled && fsEnabled;
+ }
+
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java
index 03846ae..eb7eaca 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java
@@ -383,18 +383,6 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem {
@Override
@Test
@Ignore("TODO:HDDS-2939")
- public void testTrash() throws Exception {
- }
-
- @Override
- @Test
- @Ignore("TODO:HDDS-2939")
- public void testRenameToTrashEnabled() throws Exception {
- }
-
- @Override
- @Test
- @Ignore("TODO:HDDS-2939")
public void testListStatusWithIntermediateDir() throws Exception {
}
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
index 6d7a88a..a9408a8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
@@ -35,10 +35,11 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
+import org.apache.hadoop.ozone.om.helpers.OMRatisHelper;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
-import org.apache.hadoop.ozone.om.helpers.OMRatisHelper;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
import org.apache.ratis.protocol.ClientId;
@@ -154,6 +155,11 @@ public class TrashOzoneFileSystem extends FileSystem {
// check whether the src and dst belong to the same bucket & trashroot.
OFSPath srcPath = new OFSPath(src);
OFSPath dstPath = new OFSPath(dst);
+ OmBucketInfo bucket = ozoneManager.getBucketInfo(srcPath.getVolumeName(),
+ srcPath.getBucketName());
+ if (OzoneFSUtils.isFSOptimizedBucket(bucket.getMetadata())) {
+ return renameV1(srcPath, dstPath);
+ }
Preconditions.checkArgument(srcPath.getBucketName().
equals(dstPath.getBucketName()));
Preconditions.checkArgument(srcPath.getTrashRoot().
@@ -163,14 +169,50 @@ public class TrashOzoneFileSystem extends FileSystem {
return true;
}
+ private boolean renameV1(OFSPath srcPath, OFSPath dstPath) {
+ OzoneManagerProtocolProtos.OMRequest omRequest =
+ getRenameKeyRequest(srcPath, dstPath);
+ try {
+ if(omRequest != null) {
+ submitRequest(omRequest);
+ return true;
+ }
+ return false;
+ } catch (Exception e){
+ LOG.error("couldnt send rename requestV1", e);
+ return false;
+ }
+ }
+
@Override
public boolean delete(Path path, boolean b) throws IOException {
ozoneManager.getMetrics().incNumTrashDeletes();
+ OFSPath srcPath = new OFSPath(path);
+ OmBucketInfo bucket = ozoneManager.getBucketInfo(srcPath.getVolumeName(),
+ srcPath.getBucketName());
+ if (OzoneFSUtils.isFSOptimizedBucket(bucket.getMetadata())) {
+ return deleteV1(srcPath);
+ }
DeleteIterator iterator = new DeleteIterator(path, true);
iterator.iterate();
return true;
}
+ private boolean deleteV1(OFSPath srcPath) {
+ OzoneManagerProtocolProtos.OMRequest omRequest =
+ getDeleteKeyRequest(srcPath);
+ try {
+ if(omRequest != null) {
+ submitRequest(omRequest);
+ return true;
+ }
+ return false;
+ } catch (Throwable e) {
+ LOG.error("Couldn't send delete request.", e);
+ return false;
+ }
+ }
+
@Override
public FileStatus[] listStatus(Path path) throws IOException {
ozoneManager.getMetrics().incNumTrashListStatus();
@@ -377,6 +419,41 @@ public class TrashOzoneFileSystem extends FileSystem {
}
}
+
+ private OzoneManagerProtocolProtos.OMRequest
+ getRenameKeyRequest(
+ OFSPath src, OFSPath dst) {
+ String volumeName = src.getVolumeName();
+ String bucketName = src.getBucketName();
+ String keyName = src.getKeyName();
+
+ OzoneManagerProtocolProtos.KeyArgs keyArgs =
+ OzoneManagerProtocolProtos.KeyArgs.newBuilder()
+ .setKeyName(keyName)
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .build();
+ String toKeyName = dst.getKeyName();
+ OzoneManagerProtocolProtos.RenameKeyRequest renameKeyRequest =
+ OzoneManagerProtocolProtos.RenameKeyRequest.newBuilder()
+ .setKeyArgs(keyArgs)
+ .setToKeyName(toKeyName)
+ .build();
+ OzoneManagerProtocolProtos.OMRequest omRequest =
+ null;
+ try {
+ omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder()
+ .setClientId(CLIENT_ID.toString())
+ .setUserInfo(getUserInfo())
+ .setRenameKeyRequest(renameKeyRequest)
+ .setCmdType(OzoneManagerProtocolProtos.Type.RenameKey)
+ .build();
+ } catch (IOException e) {
+ LOG.error("Couldn't get userinfo", e);
+ }
+ return omRequest;
+ }
+
private class RenameIterator extends OzoneListingIterator {
private final String srcPath;
private final String dstPath;
@@ -408,40 +485,37 @@ public class TrashOzoneFileSystem extends FileSystem {
}
return true;
}
+ }
- private OzoneManagerProtocolProtos.OMRequest
- getRenameKeyRequest(
- OFSPath src, OFSPath dst) {
- String volumeName = src.getVolumeName();
- String bucketName = src.getBucketName();
- String keyName = src.getKeyName();
-
- OzoneManagerProtocolProtos.KeyArgs keyArgs =
- OzoneManagerProtocolProtos.KeyArgs.newBuilder()
- .setKeyName(keyName)
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .build();
- String toKeyName = dst.getKeyName();
- OzoneManagerProtocolProtos.RenameKeyRequest renameKeyRequest =
- OzoneManagerProtocolProtos.RenameKeyRequest.newBuilder()
- .setKeyArgs(keyArgs)
- .setToKeyName(toKeyName)
- .build();
- OzoneManagerProtocolProtos.OMRequest omRequest =
- null;
- try {
- omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder()
- .setClientId(CLIENT_ID.toString())
- .setUserInfo(getUserInfo())
- .setRenameKeyRequest(renameKeyRequest)
- .setCmdType(OzoneManagerProtocolProtos.Type.RenameKey)
+ private OzoneManagerProtocolProtos.OMRequest getDeleteKeyRequest(
+ OFSPath srcPath) {
+ String volume = srcPath.getVolumeName();
+ String bucket = srcPath.getBucketName();
+ String key = srcPath.getKeyName();
+ OzoneManagerProtocolProtos.KeyArgs keyArgs =
+ OzoneManagerProtocolProtos.KeyArgs.newBuilder()
+ .setKeyName(key)
+ .setVolumeName(volume)
+ .setBucketName(bucket)
+ .setRecursive(true)
.build();
- } catch (IOException e) {
- LOG.error("Couldn't get userinfo", e);
- }
- return omRequest;
+ OzoneManagerProtocolProtos.DeleteKeyRequest deleteKeyRequest =
+ OzoneManagerProtocolProtos.DeleteKeyRequest.newBuilder()
+ .setKeyArgs(keyArgs).build();
+ OzoneManagerProtocolProtos.OMRequest omRequest =
+ null;
+ try {
+ omRequest =
+ OzoneManagerProtocolProtos.OMRequest.newBuilder()
+ .setClientId(CLIENT_ID.toString())
+ .setUserInfo(getUserInfo())
+ .setDeleteKeyRequest(deleteKeyRequest)
+ .setCmdType(OzoneManagerProtocolProtos.Type.DeleteKey)
+ .build();
+ } catch (IOException e) {
+ LOG.error("Couldn't get userinfo", e);
}
+ return omRequest;
}
private class DeleteIterator extends OzoneListingIterator {
@@ -467,7 +541,7 @@ public class TrashOzoneFileSystem extends FileSystem {
for (String keyPath : keyPathList) {
OFSPath path = new OFSPath(keyPath);
OzoneManagerProtocolProtos.OMRequest omRequest =
- getDeleteKeyRequest(path);
+ getDeleteKeysRequest(path);
try {
ozoneManager.getMetrics().incNumTrashFilesDeletes();
submitRequest(omRequest);
@@ -479,7 +553,7 @@ public class TrashOzoneFileSystem extends FileSystem {
}
private OzoneManagerProtocolProtos.OMRequest
- getDeleteKeyRequest(
+ getDeleteKeysRequest(
OFSPath keyPath) {
String volumeName = keyPath.getVolumeName();
String bucketName = keyPath.getBucketName();
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
index 150108c..125934e 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
@@ -48,12 +48,12 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenRenewer;
@@ -548,17 +548,6 @@ public class BasicOzoneClientAdapterImpl implements OzoneClientAdapter {
@Override
public boolean isFSOptimizedBucket() {
- // layout version V1 represents optimized FS path
- boolean layoutVersionEnabled =
- StringUtils.equalsIgnoreCase(
- OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1,
- bucket.getMetadata()
- .get(OMConfigKeys.OZONE_OM_LAYOUT_VERSION));
-
- boolean fsEnabled =
- Boolean.parseBoolean(bucket.getMetadata()
- .get(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS));
-
- return layoutVersionEnabled && fsEnabled;
+ return OzoneFSUtils.isFSOptimizedBucket(bucket.getMetadata());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org