You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/05/09 02:42:15 UTC
[21/50] hadoop git commit: HDFS-5640. Add snapshot methods to
FileContext. Contributed by Rakesh R.
HDFS-5640. Add snapshot methods to FileContext. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/70505423
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/70505423
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/70505423
Branch: refs/heads/YARN-2928
Commit: 70505423297d264d469d4fa12b738540684a16d5
Parents: b07dc2c
Author: cnauroth <cn...@apache.org>
Authored: Fri May 8 11:20:55 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 8 17:32:49 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/AbstractFileSystem.java | 30 +++++
.../java/org/apache/hadoop/fs/FileContext.java | 101 +++++++++++++++
.../java/org/apache/hadoop/fs/FilterFs.java | 18 +++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../main/java/org/apache/hadoop/fs/Hdfs.java | 18 +++
.../snapshot/TestFileContextSnapshot.java | 124 +++++++++++++++++++
6 files changed, 293 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70505423/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index 959d9d5..7af5fa7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -1191,6 +1191,36 @@ public abstract class AbstractFileSystem {
+ " doesn't support removeXAttr");
}
+ /**
+ * The specification of this method matches that of
+ * {@link FileContext#createSnapshot(Path, String)}.
+ */
+ public Path createSnapshot(final Path path, final String snapshotName)
+ throws IOException {
+ throw new UnsupportedOperationException(getClass().getSimpleName()
+ + " doesn't support createSnapshot");
+ }
+
+ /**
+ * The specification of this method matches that of
+ * {@link FileContext#renameSnapshot(Path, String, String)}.
+ */
+ public void renameSnapshot(final Path path, final String snapshotOldName,
+ final String snapshotNewName) throws IOException {
+ throw new UnsupportedOperationException(getClass().getSimpleName()
+ + " doesn't support renameSnapshot");
+ }
+
+ /**
+ * The specification of this method matches that of
+ * {@link FileContext#deleteSnapshot(Path, String)}.
+ */
+ public void deleteSnapshot(final Path snapshotDir, final String snapshotName)
+ throws IOException {
+ throw new UnsupportedOperationException(getClass().getSimpleName()
+ + " doesn't support deleteSnapshot");
+ }
+
@Override //Object
public int hashCode() {
return myUri.hashCode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70505423/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index ea3f896..16cb591 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -2564,4 +2564,105 @@ public class FileContext {
}
}.resolve(this, absF);
}
+
+ /**
+ * Create a snapshot with a default name.
+ *
+ * @param path The directory where snapshots will be taken.
+ * @return the snapshot path.
+ *
+ * @throws IOException If an I/O error occurred
+ *
+ * <p>Exceptions applicable to file systems accessed over RPC:
+ * @throws RpcClientException If an exception occurred in the RPC client
+ * @throws RpcServerException If an exception occurred in the RPC server
+ * @throws UnexpectedServerException If server implementation throws
+ * undeclared exception to RPC server
+ */
+ public final Path createSnapshot(Path path) throws IOException {
+ return createSnapshot(path, null);
+ }
+
+ /**
+ * Create a snapshot.
+ *
+ * @param path The directory where snapshots will be taken.
+ * @param snapshotName The name of the snapshot
+ * @return the snapshot path.
+ *
+ * @throws IOException If an I/O error occurred
+ *
+ * <p>Exceptions applicable to file systems accessed over RPC:
+ * @throws RpcClientException If an exception occurred in the RPC client
+ * @throws RpcServerException If an exception occurred in the RPC server
+ * @throws UnexpectedServerException If server implementation throws
+ * undeclared exception to RPC server
+ */
+ public Path createSnapshot(final Path path, final String snapshotName)
+ throws IOException {
+ final Path absF = fixRelativePart(path);
+ return new FSLinkResolver<Path>() {
+
+ @Override
+ public Path next(final AbstractFileSystem fs, final Path p)
+ throws IOException {
+ return fs.createSnapshot(p, snapshotName);
+ }
+ }.resolve(this, absF);
+ }
+
+ /**
+ * Rename a snapshot.
+ *
+ * @param path The directory path where the snapshot was taken
+ * @param snapshotOldName Old name of the snapshot
+ * @param snapshotNewName New name of the snapshot
+ *
+ * @throws IOException If an I/O error occurred
+ *
+ * <p>Exceptions applicable to file systems accessed over RPC:
+ * @throws RpcClientException If an exception occurred in the RPC client
+ * @throws RpcServerException If an exception occurred in the RPC server
+ * @throws UnexpectedServerException If server implementation throws
+ * undeclared exception to RPC server
+ */
+ public void renameSnapshot(final Path path, final String snapshotOldName,
+ final String snapshotNewName) throws IOException {
+ final Path absF = fixRelativePart(path);
+ new FSLinkResolver<Void>() {
+ @Override
+ public Void next(final AbstractFileSystem fs, final Path p)
+ throws IOException {
+ fs.renameSnapshot(p, snapshotOldName, snapshotNewName);
+ return null;
+ }
+ }.resolve(this, absF);
+ }
+
+ /**
+ * Delete a snapshot of a directory.
+ *
+ * @param path The directory that the to-be-deleted snapshot belongs to
+ * @param snapshotName The name of the snapshot
+ *
+ * @throws IOException If an I/O error occurred
+ *
+ * <p>Exceptions applicable to file systems accessed over RPC:
+ * @throws RpcClientException If an exception occurred in the RPC client
+ * @throws RpcServerException If an exception occurred in the RPC server
+ * @throws UnexpectedServerException If server implementation throws
+ * undeclared exception to RPC server
+ */
+ public void deleteSnapshot(final Path path, final String snapshotName)
+ throws IOException {
+ final Path absF = fixRelativePart(path);
+ new FSLinkResolver<Void>() {
+ @Override
+ public Void next(final AbstractFileSystem fs, final Path p)
+ throws IOException {
+ fs.deleteSnapshot(p, snapshotName);
+ return null;
+ }
+ }.resolve(this, absF);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70505423/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
index 4f28a9a..f6a6f23 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
@@ -371,4 +371,22 @@ public abstract class FilterFs extends AbstractFileSystem {
public void removeXAttr(Path path, String name) throws IOException {
myFs.removeXAttr(path, name);
}
+
+ @Override
+ public Path createSnapshot(final Path path, final String snapshotName)
+ throws IOException {
+ return myFs.createSnapshot(path, snapshotName);
+ }
+
+ @Override
+ public void renameSnapshot(final Path path, final String snapshotOldName,
+ final String snapshotNewName) throws IOException {
+ myFs.renameSnapshot(path, snapshotOldName, snapshotNewName);
+ }
+
+ @Override
+ public void deleteSnapshot(final Path path, final String snapshotName)
+ throws IOException {
+ myFs.deleteSnapshot(path, snapshotName);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70505423/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e22f961..efce95b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -525,6 +525,8 @@ Release 2.8.0 - UNRELEASED
HDFS-7559. Create unit test to automatically compare HDFS related classes
and hdfs-default.xml. (Ray Chiang via asuresh)
+ HDFS-5640. Add snapshot methods to FileContext. (Rakesh R via cnauroth)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70505423/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
index aaaff25..b776849 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
@@ -495,4 +495,22 @@ public class Hdfs extends AbstractFileSystem {
throws InvalidToken, IOException {
dfs.cancelDelegationToken((Token<DelegationTokenIdentifier>) token);
}
+
+ @Override
+ public Path createSnapshot(final Path path, final String snapshotName)
+ throws IOException {
+ return new Path(dfs.createSnapshot(getUriPath(path), snapshotName));
+ }
+
+ @Override
+ public void renameSnapshot(final Path path, final String snapshotOldName,
+ final String snapshotNewName) throws IOException {
+ dfs.renameSnapshot(getUriPath(path), snapshotOldName, snapshotNewName);
+ }
+
+ @Override
+ public void deleteSnapshot(final Path snapshotDir, final String snapshotName)
+ throws IOException {
+ dfs.deleteSnapshot(getUriPath(snapshotDir), snapshotName);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70505423/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileContextSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileContextSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileContextSnapshot.java
new file mode 100644
index 0000000..dcabed7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileContextSnapshot.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode.snapshot;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.SnapshotException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestFileContextSnapshot {
+
+ private static final short REPLICATION = 3;
+ private static final int BLOCKSIZE = 1024;
+ private static final long SEED = 0;
+ private Configuration conf;
+ private MiniDFSCluster cluster;
+ private FileContext fileContext;
+ private DistributedFileSystem dfs;
+
+ private final String snapshotRoot = "/snapshot";
+ private final Path filePath = new Path(snapshotRoot, "file1");
+ private Path snapRootPath;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION)
+ .build();
+ cluster.waitActive();
+
+ fileContext = FileContext.getFileContext(conf);
+ dfs = (DistributedFileSystem) cluster.getFileSystem();
+ snapRootPath = new Path(snapshotRoot);
+ dfs.mkdirs(snapRootPath);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test(timeout = 60000)
+ public void testCreateAndDeleteSnapshot() throws Exception {
+ DFSTestUtil.createFile(dfs, filePath, BLOCKSIZE, REPLICATION, SEED);
+ // disallow snapshot on dir
+ dfs.disallowSnapshot(snapRootPath);
+ try {
+ fileContext.createSnapshot(snapRootPath, "s1");
+ } catch (SnapshotException e) {
+ GenericTestUtils.assertExceptionContains(
+ "Directory is not a snapshottable directory: " + snapRootPath, e);
+ }
+
+ // allow snapshot on dir
+ dfs.allowSnapshot(snapRootPath);
+ Path ssPath = fileContext.createSnapshot(snapRootPath, "s1");
+ assertTrue("Failed to create snapshot", dfs.exists(ssPath));
+ fileContext.deleteSnapshot(snapRootPath, "s1");
+ assertFalse("Failed to delete snapshot", dfs.exists(ssPath));
+ }
+
+ /**
+ * Test FileStatus of snapshot file before/after rename
+ */
+ @Test(timeout = 60000)
+ public void testRenameSnapshot() throws Exception {
+ DFSTestUtil.createFile(dfs, filePath, BLOCKSIZE, REPLICATION, SEED);
+ dfs.allowSnapshot(snapRootPath);
+ // Create snapshot for sub1
+ Path snapPath1 = fileContext.createSnapshot(snapRootPath, "s1");
+ Path ssPath = new Path(snapPath1, filePath.getName());
+ assertTrue("Failed to create snapshot", dfs.exists(ssPath));
+ FileStatus statusBeforeRename = dfs.getFileStatus(ssPath);
+
+ // Rename the snapshot
+ fileContext.renameSnapshot(snapRootPath, "s1", "s2");
+ // <sub1>/.snapshot/s1/file1 should no longer exist
+ assertFalse("Old snapshot still exists after rename!", dfs.exists(ssPath));
+ Path snapshotRoot = SnapshotTestHelper.getSnapshotRoot(snapRootPath, "s2");
+ ssPath = new Path(snapshotRoot, filePath.getName());
+
+ // Instead, <sub1>/.snapshot/s2/file1 should exist
+ assertTrue("Snapshot doesn't exists!", dfs.exists(ssPath));
+ FileStatus statusAfterRename = dfs.getFileStatus(ssPath);
+
+ // FileStatus of the snapshot should not change except the path
+ assertFalse("Filestatus of the snapshot matches",
+ statusBeforeRename.equals(statusAfterRename));
+ statusBeforeRename.setPath(statusAfterRename.getPath());
+ assertEquals("FileStatus of the snapshot mismatches!",
+ statusBeforeRename.toString(), statusAfterRename.toString());
+ }
+}
\ No newline at end of file