You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2022/11/17 00:37:20 UTC
[ozone] branch HDDS-6517-Snapshot updated: HDDS-7375. [Snapshot] Implement `ozone fs` command for creating snapshot (#3937)
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-6517-Snapshot
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-6517-Snapshot by this push:
new f1f1c0ba42 HDDS-7375. [Snapshot] Implement `ozone fs` command for creating snapshot (#3937)
f1f1c0ba42 is described below
commit f1f1c0ba427b498ca93542a3838cd96278550e0b
Author: Hemant Kumar <he...@gmail.com>
AuthorDate: Wed Nov 16 16:37:13 2022 -0800
HDDS-7375. [Snapshot] Implement `ozone fs` command for creating snapshot (#3937)
---
.../apache/hadoop/ozone/client/ObjectStore.java | 5 +-
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 119 +++++++++++++++++++++
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 10 ++
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 10 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 9 ++
.../fs/ozone/BasicRootedOzoneFileSystem.java | 9 ++
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 2 +
7 files changed, 161 insertions(+), 3 deletions(-)
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index c8e1c1f54c..55243ed31b 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -538,8 +538,9 @@ public class ObjectStore {
/**
* Create snapshot.
- * @param name name to be used
- * @param snapshotPath snapshotPath to be used
+ * @param volumeName vol to be used
+ * @param bucketName bucket to be used
+ * @param snapshotName name to be used
* @return name used
* @throws IOException
*/
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java
new file mode 100644
index 0000000000..91f5251ed2
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.util.UUID;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test client-side CRUD snapshot operations with Ozone Manager.
+ */
+public class TestOzoneFsSnapshot {
+ // Set the timeout for every test.
+ @Rule
+ public Timeout testTimeout = Timeout.seconds(300);
+
+ private static MiniOzoneCluster cluster;
+ private static final String OM_SERVICE_ID = "om-service-test1";
+ private OzoneConfiguration clientConf;
+ private static OzoneManager ozoneManager;
+
+ @BeforeClass
+ public static void initClass() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+
+ // Start the cluster
+ cluster = MiniOzoneCluster.newOMHABuilder(conf)
+ .setClusterId(UUID.randomUUID().toString())
+ .setScmId(UUID.randomUUID().toString())
+ .setOMServiceId(OM_SERVICE_ID)
+ .setNumOfOzoneManagers(1)
+ .build();
+ cluster.waitForClusterToBeReady();
+ ozoneManager = cluster.getOzoneManager();
+ }
+
+ @Before
+ public void init() {
+ String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + OM_SERVICE_ID;
+ clientConf = new OzoneConfiguration(cluster.getConf());
+ clientConf.set(FS_DEFAULT_NAME_KEY, hostPrefix);
+ }
+
+ @AfterClass
+ public static void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testCreateSnapshot() throws Exception {
+ String volume = "vol1";
+ String bucket = "bucket1";
+ String testVolBucket = OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket;
+ String snapshotName = "snap1";
+ String testKey = testVolBucket + "/key1";
+
+ OzoneFsShell shell = new OzoneFsShell(clientConf);
+ try {
+ // Create volume and bucket
+ int res = ToolRunner.run(shell,
+ new String[]{"-mkdir", "-p", testVolBucket});
+ assertEquals(0, res);
+ // Create key
+ ToolRunner.run(shell, new String[]{"-touch", testKey});
+ assertEquals(0, res);
+ // List the bucket to make sure that bucket exists.
+ ToolRunner.run(shell, new String[]{"-ls", testVolBucket});
+ assertEquals(0, res);
+
+ res = ToolRunner.run(shell,
+ new String[]{"-createSnapshot", testVolBucket, snapshotName});
+ // Asserts that create request succeeded
+ assertEquals(0, res);
+
+ SnapshotInfo snapshotInfo = ozoneManager
+ .getMetadataManager()
+ .getSnapshotInfoTable()
+ .get(SnapshotInfo.getTableKey(volume, bucket, snapshotName));
+
+ // Assert that snapshot exists in RocksDB.
+ // We can't use list or valid if snapshot directory exists because DB
+ // transaction might not be flushed by the time.
+ Assert.assertNotNull(snapshotInfo);
+ } finally {
+ shell.close();
+ }
+ }
+}
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
index bec00e9299..16fc844925 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OFSPath;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.client.ObjectStore;
@@ -609,4 +610,13 @@ public class BasicOzoneClientAdapterImpl implements OzoneClientAdapter {
length, combineMode, ozoneClient.getObjectStore().getClientProxy());
}
+
+ @Override
+ public String createSnapshot(String pathStr, String snapshotName)
+ throws IOException {
+ OFSPath ofsPath = new OFSPath(pathStr);
+ return objectStore.createSnapshot(ofsPath.getVolumeName(),
+ ofsPath.getBucketName(),
+ snapshotName);
+ }
}
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
index 04683ff760..1b7e2d8264 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
@@ -74,6 +74,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZ
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
@@ -870,6 +871,14 @@ public class BasicOzoneFileSystem extends FileSystem {
return new OzoneFileStatusIterator<>(f);
}
+ @Override
+ public Path createSnapshot(Path path, String snapshotName)
+ throws IOException {
+ String snapshot = adapter.createSnapshot(pathToKey(path), snapshotName);
+ return new Path(path,
+ OM_SNAPSHOT_INDICATOR + OZONE_URI_DELIMITER + snapshot);
+ }
+
/**
* A private class implementation for iterating list of file status.
*
@@ -1183,5 +1192,4 @@ public class BasicOzoneFileSystem extends FileSystem {
}
return new LocatedFileStatus(fileStatus, blockLocations);
}
-
}
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index 51ba277bf7..47612f5b4a 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -1118,4 +1118,13 @@ public class BasicRootedOzoneClientAdapterImpl
length, combineMode, ozoneClient.getObjectStore().getClientProxy());
}
+
+ @Override
+ public String createSnapshot(String pathStr, String snapshotName)
+ throws IOException {
+ OFSPath ofsPath = new OFSPath(pathStr);
+ return proxy.createSnapshot(ofsPath.getVolumeName(),
+ ofsPath.getBucketName(),
+ snapshotName);
+ }
}
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index 28491d66fd..4e8f28ea7b 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -73,6 +73,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZ
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_EMPTY;
@@ -441,6 +442,14 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
}
}
+ @Override
+ public Path createSnapshot(Path path, String snapshotName)
+ throws IOException {
+ String snapshot = adapter.createSnapshot(pathToKey(path), snapshotName);
+ return new Path(path,
+ OM_SNAPSHOT_INDICATOR + OZONE_URI_DELIMITER + snapshot);
+ }
+
private class DeleteIterator extends OzoneListingIterator {
private final boolean recursive;
private final OzoneBucket bucket;
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
index 31bf351f01..6019996837 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
@@ -81,4 +81,6 @@ public interface OzoneClientAdapter {
boolean isFSOptimizedBucket();
FileChecksum getFileChecksum(String keyName, long length) throws IOException;
+
+ String createSnapshot(String pathStr, String snapshotName) throws IOException;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org