You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2013/01/23 03:48:02 UTC
svn commit: r1437256 [2/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/main/java/org/apache/hadoop/hdfs/server/namenode/snaps...
Added: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java?rev=1437256&view=auto
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java (added)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java Wed Jan 23 02:48:01 2013
@@ -0,0 +1,351 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode.snapshot;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
+import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
+import org.apache.hadoop.hdfs.server.namenode.FSImageFormat.Loader;
+import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
+import org.apache.hadoop.hdfs.server.namenode.INode;
+import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot.Diff;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot.SnapshotDiff;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.Root;
+import org.apache.hadoop.hdfs.util.ReadOnlyList;
+
+/**
+ * A helper class defining static methods for reading/writing snapshot related
+ * information from/to FSImage.
+ */
+public class SnapshotFSImageFormat {
+
+ /**
+ * Save snapshots and snapshot quota for a snapshottable directory.
+ * @param current The directory that the snapshots belongs to.
+ * @param out The {@link DataOutputStream} to write.
+ * @throws IOException
+ */
+ public static void saveSnapshots(INodeDirectorySnapshottable current,
+ DataOutputStream out) throws IOException {
+ // list of snapshots in snapshotsByNames
+ ReadOnlyList<Snapshot> snapshots = current.getSnapshotsByNames();
+ out.writeInt(snapshots.size());
+ for (Snapshot ss : snapshots) {
+ // write the snapshot
+ ss.write(out);
+ }
+ // snapshot quota
+ out.writeInt(current.getSnapshotQuota());
+ }
+
+ /**
+ * Save SnapshotDiff list for an INodeDirectoryWithSnapshot.
+ * @param sNode The directory that the SnapshotDiff list belongs to.
+ * @param out The {@link DataOutputStream} to write.
+ */
+ public static void saveSnapshotDiffs(INodeDirectoryWithSnapshot sNode,
+ DataOutputStream out) throws IOException {
+ // # of SnapshotDiff
+ List<SnapshotDiff> diffs = sNode.getSnapshotDiffs();
+ // Record the SnapshotDiff in reversed order, so that we can find the
+ // correct reference for INodes in the created list when loading the
+ // FSImage
+ out.writeInt(diffs.size());
+ for (int i = diffs.size() - 1; i >= 0; i--) {
+ SnapshotDiff sdiff = diffs.get(i);
+ sdiff.write(out);
+ }
+ }
+
+ /**
+ * Load a node stored in the created list from fsimage.
+ * @param createdNodeName The name of the created node.
+ * @param parent The directory that the created list belongs to.
+ * @return The created node.
+ */
+ private static INode loadCreated(byte[] createdNodeName,
+ INodeDirectoryWithSnapshot parent) throws IOException {
+ // the INode in the created list should be a reference to another INode
+ // in posterior SnapshotDiffs or one of the current children
+ for (SnapshotDiff postDiff : parent.getSnapshotDiffs()) {
+ INode created = findCreated(createdNodeName, postDiff.getDiff());
+ if (created != null) {
+ return created;
+ } // else go to the next SnapshotDiff
+ }
+ // use the current child
+ INode currentChild = parent.getChild(createdNodeName, null);
+ if (currentChild == null) {
+ throw new IOException("Cannot find an INode associated with the INode "
+ + DFSUtil.bytes2String(createdNodeName)
+ + " in created list while loading FSImage.");
+ }
+ return currentChild;
+ }
+
+ /**
+ * Search the given {@link Diff} to find an inode matching the specific name.
+ * @param createdNodeName The name of the node for searching.
+ * @param diff The given {@link Diff} where to search the node.
+ * @return The matched inode. Return null if no matched inode can be found.
+ */
+ private static INode findCreated(byte[] createdNodeName, Diff diff) {
+ INode c = diff.searchCreated(createdNodeName);
+ INode d = diff.searchDeleted(createdNodeName);
+ if (c == null && d != null) {
+ // if an INode with the same name is only contained in the deleted
+ // list, then the node should be the snapshot copy of a deleted
+ // node, and the node in the created list should be its reference
+ return d;
+ } else if (c != null && d != null) {
+ // in a posterior SnapshotDiff, if the created/deleted lists both
+ // contains nodes with the same name (c & d), there are two
+ // possibilities:
+ //
+ // 1) c and d are used to represent a modification, and
+ // 2) d indicates the deletion of the node, while c was originally
+ // contained in the created list of a later snapshot, but c was
+ // moved here because of the snapshot deletion.
+ //
+ // For case 1), c and d should be both INodeFile and should share
+ // the same blockInfo list.
+ if (c.isFile()
+ && ((INodeFile) c).getBlocks() == ((INodeFile) d).getBlocks()) {
+ return c;
+ } else {
+ return d;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Load the created list from fsimage.
+ * @param parent The directory that the created list belongs to.
+ * @param in The {@link DataInputStream} to read.
+ * @return The created list.
+ */
+ private static List<INode> loadCreatedList(INodeDirectoryWithSnapshot parent,
+ DataInputStream in) throws IOException {
+ // read the size of the created list
+ int createdSize = in.readInt();
+ List<INode> createdList = new ArrayList<INode>(createdSize);
+ for (int i = 0; i < createdSize; i++) {
+ byte[] createdNodeName = new byte[in.readShort()];
+ in.readFully(createdNodeName);
+ INode created = loadCreated(createdNodeName, parent);
+ createdList.add(created);
+ }
+ return createdList;
+ }
+
+ /**
+ * Load the deleted list from the fsimage.
+ *
+ * @param parent The directory that the deleted list belongs to.
+ * @param createdList The created list associated with the deleted list in
+ * the same Diff.
+ * @param in The {@link DataInputStream} to read.
+ * @param loader The {@link Loader} instance. Used to call the
+ * {@link Loader#loadINode(DataInputStream)} method.
+ * @return The deleted list.
+ */
+ private static List<INode> loadDeletedList(INodeDirectoryWithSnapshot parent,
+ List<INode> createdList, DataInputStream in, FSImageFormat.Loader loader)
+ throws IOException {
+ int deletedSize = in.readInt();
+ List<INode> deletedList = new ArrayList<INode>(deletedSize);
+ for (int i = 0; i < deletedSize; i++) {
+ byte[] deletedNodeName = new byte[in.readShort()];
+ in.readFully(deletedNodeName);
+ INode deleted = loader.loadINode(in);
+ deleted.setLocalName(deletedNodeName);
+ deletedList.add(deleted);
+ // set parent: the parent field of an INode in the deleted list is not
+ // useful, but set the parent here to be consistent with the original
+ // fsdir tree.
+ deleted.setParent(parent);
+ if (deleted instanceof INodeFile
+ && ((INodeFile) deleted).getBlocks() == null) {
+ // if deleted is an INodeFile, and its blocks is null, then deleted
+ // must be an INodeFileWithLink, and we need to rebuild its next link
+ int c = Collections.binarySearch(createdList, deletedNodeName);
+ if (c < 0) {
+ throw new IOException(
+ "Cannot find the INode linked with the INode "
+ + DFSUtil.bytes2String(deletedNodeName)
+ + " in deleted list while loading FSImage.");
+ }
+ // deleted must be an INodeFileSnapshot
+ INodeFileSnapshot deletedWithLink = (INodeFileSnapshot) deleted;
+ INodeFile cNode = (INodeFile) createdList.get(c);
+ INodeFileWithSnapshot cNodeWithLink = (INodeFileWithSnapshot) cNode;
+ deletedWithLink.setBlocks(cNode.getBlocks());
+ // insert deleted into the circular list
+ cNodeWithLink.insertBefore(deletedWithLink);
+ }
+ }
+ return deletedList;
+ }
+
+ /**
+ * Load snapshots and snapshotQuota for a Snapshottable directory.
+ * @param snapshottableParent The snapshottable directory for loading.
+ * @param numSnapshots The number of snapshots that the directory has.
+ * @param in The {@link DataInputStream} instance to read.
+ * @param loader The {@link Loader} instance that this loading procedure is
+ * using.
+ */
+ public static void loadSnapshotList(
+ INodeDirectorySnapshottable snapshottableParent, int numSnapshots,
+ DataInputStream in, FSImageFormat.Loader loader) throws IOException {
+ for (int i = 0; i < numSnapshots; i++) {
+ // read snapshots
+ Snapshot ss = loadSnapshot(snapshottableParent, in, loader);
+ snapshottableParent.addSnapshot(ss);
+ }
+ int snapshotQuota = in.readInt();
+ snapshottableParent.setSnapshotQuota(snapshotQuota);
+ }
+
+ /**
+ * Load a {@link Snapshot} from fsimage.
+ * @param parent The directory that the snapshot belongs to.
+ * @param in The {@link DataInputStream} instance to read.
+ * @param loader The {@link Loader} instance that this loading procedure is
+ * using.
+ * @return The snapshot.
+ */
+ private static Snapshot loadSnapshot(INodeDirectorySnapshottable parent,
+ DataInputStream in, FSImageFormat.Loader loader) throws IOException {
+ int snapshotId = in.readInt();
+ byte[] snapshotName = new byte[in.readShort()];
+ in.readFully(snapshotName);
+ INode rootNode = loader.loadINode(in);
+ rootNode.setLocalName(snapshotName);
+ rootNode.setParent(parent);
+ return new Snapshot(snapshotId, (INodeDirectory) rootNode);
+ }
+
+ /**
+ * Load the {@link SnapshotDiff} list for the INodeDirectoryWithSnapshot
+ * directory.
+ * @param snapshottableParent The snapshottable directory for loading.
+ * @param numSnapshotDiffs The number of {@link SnapshotDiff} that the
+ * directory has.
+ * @param in The {@link DataInputStream} instance to read.
+ * @param loader The {@link Loader} instance that this loading procedure is
+ * using.
+ */
+ public static void loadSnapshotDiffList(
+ INodeDirectoryWithSnapshot parentWithSnapshot, int numSnapshotDiffs,
+ DataInputStream in, FSImageFormat.Loader loader)
+ throws IOException {
+ for (int i = 0; i < numSnapshotDiffs; i++) {
+ SnapshotDiff diff = loadSnapshotDiff(parentWithSnapshot, in, loader);
+ parentWithSnapshot.insertDiff(diff);
+ }
+ }
+
+ /**
+ * Use the given full path to a {@link Root} directory to find the
+ * associated snapshot.
+ */
+ private static Snapshot findSnapshot(String sRootFullPath, FSDirectory fsdir)
+ throws IOException {
+ // find the root
+ INode root = fsdir.getINode(sRootFullPath);
+ INodeDirectorySnapshottable snapshotRoot = INodeDirectorySnapshottable
+ .valueOf(root.getParent(), root.getParent().getFullPathName());
+ // find the snapshot
+ return snapshotRoot.getSnapshot(root.getLocalNameBytes());
+ }
+
+ /**
+ * Load the snapshotINode field of {@link SnapshotDiff}.
+ * @param snapshot The Snapshot associated with the {@link SnapshotDiff}.
+ * @param in The {@link DataInputStream} to read.
+ * @param loader The {@link Loader} instance that this loading procedure is
+ * using.
+ * @return The snapshotINode.
+ */
+ private static INodeDirectory loadSnapshotINodeInSnapshotDiff(
+ Snapshot snapshot, DataInputStream in, FSImageFormat.Loader loader)
+ throws IOException {
+ // read the boolean indicating whether snapshotINode == Snapshot.Root
+ boolean useRoot = in.readBoolean();
+ if (useRoot) {
+ return snapshot.getRoot();
+ } else {
+ // another boolean is used to indicate whether snapshotINode is non-null
+ if (in.readBoolean()) {
+ byte[] localName = new byte[in.readShort()];
+ in.readFully(localName);
+ INodeDirectory snapshotINode = (INodeDirectory) loader.loadINode(in);
+ snapshotINode.setLocalName(localName);
+ return snapshotINode;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Load {@link SnapshotDiff} from fsimage.
+ * @param parent The directory that the SnapshotDiff belongs to.
+ * @param in The {@link DataInputStream} instance to read.
+ * @param loader The {@link Loader} instance that this loading procedure is
+ * using.
+ * @return A {@link SnapshotDiff}.
+ */
+ private static SnapshotDiff loadSnapshotDiff(
+ INodeDirectoryWithSnapshot parent, DataInputStream in,
+ FSImageFormat.Loader loader) throws IOException {
+ // 1. Load SnapshotDiff#childrenSize
+ int childrenSize = in.readInt();
+ // 2. Read the full path of the Snapshot's Root, identify
+ // SnapshotDiff#Snapshot
+ Snapshot snapshot = findSnapshot(FSImageSerialization.readString(in),
+ loader.getFSDirectoryInLoading());
+
+ // 3. Load SnapshotDiff#snapshotINode
+ INodeDirectory snapshotINode = loadSnapshotINodeInSnapshotDiff(snapshot,
+ in, loader);
+
+ // 4. Load the created list in SnapshotDiff#Diff
+ List<INode> createdList = loadCreatedList(parent, in);
+
+ // 5. Load the deleted list in SnapshotDiff#Diff
+ List<INode> deletedList = loadDeletedList(parent, createdList, in, loader);
+
+ // 6. Compose the SnapshotDiff
+ SnapshotDiff sdiff = parent.new SnapshotDiff(snapshot, childrenSize,
+ snapshotINode, parent.getSnapshotDiffs().isEmpty() ? null : parent
+ .getSnapshotDiffs().get(0), createdList, deletedList);
+ return sdiff;
+ }
+
+}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java?rev=1437256&r1=1437255&r2=1437256&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java Wed Jan 23 02:48:01 2013
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.namenode.snapshot;
+import java.io.DataInput;
+import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -48,7 +50,7 @@ public class SnapshotManager implements
private final AtomicInteger numSnapshottableDirs = new AtomicInteger();
private final AtomicInteger numSnapshots = new AtomicInteger();
- private int snapshotID = 0;
+ private int snapshotCounter = 0;
/** All snapshottable directories in the namesystem. */
private final List<INodeDirectorySnapshottable> snapshottables
@@ -117,10 +119,10 @@ public class SnapshotManager implements
final INodesInPath i = fsdir.getMutableINodesInPath(path);
final INodeDirectorySnapshottable srcRoot
= INodeDirectorySnapshottable.valueOf(i.getLastINode(), path);
- srcRoot.addSnapshot(snapshotID, snapshotName);
+ srcRoot.addSnapshot(snapshotCounter, snapshotName);
//create success, update id
- snapshotID++;
+ snapshotCounter++;
numSnapshots.getAndIncrement();
}
@@ -181,6 +183,26 @@ public class SnapshotManager implements
}
/**
+ * Write {@link #snapshotCounter}, {@link #numSnapshots}, and
+ * {@link #numSnapshottableDirs} to the DataOutput.
+ */
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(snapshotCounter);
+ out.writeInt(numSnapshots.get());
+ out.writeInt(numSnapshottableDirs.get());
+ }
+
+ /**
+ * Read values of {@link #snapshotCounter}, {@link #numSnapshots}, and
+ * {@link #numSnapshottableDirs} from the DataInput
+ */
+ public void read(DataInput in) throws IOException {
+ snapshotCounter = in.readInt();
+ numSnapshots.set(in.readInt());
+ numSnapshottableDirs.set(in.readInt());
+ }
+
+ /**
* @return All the current snapshottable directories
*/
public SnapshottableDirectoryStatus[] getSnapshottableDirListing() {
Added: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java?rev=1437256&view=auto
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java (added)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java Wed Jan 23 02:48:01 2013
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
+import org.apache.hadoop.hdfs.util.Canceler;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test FSImage save/load when Snapshot is supported
+ */
+public class TestFSImageWithSnapshot {
+ static final long seed = 0;
+ static final short REPLICATION = 3;
+ static final long BLOCKSIZE = 1024;
+ static final long txid = 1;
+
+ private final Path dir = new Path("/TestSnapshot");
+ private static String testDir =
+ System.getProperty("test.build.data", "build/test/data");
+
+ Configuration conf;
+ MiniDFSCluster cluster;
+ FSNamesystem fsn;
+ DistributedFileSystem hdfs;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION)
+ .build();
+ cluster.waitActive();
+ fsn = cluster.getNamesystem();
+ hdfs = cluster.getFileSystem();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Testing steps:
+ * <pre>
+ * 1. Creating/modifying directories/files while snapshots are being taken.
+ * 2. Dump the FSDirectory tree of the namesystem.
+ * 3. Save the namesystem to a temp file (FSImage saving).
+ * 4. Restart the cluster and format the namesystem.
+ * 5. Load the namesystem from the temp file (FSImage loading).
+ * 6. Dump the FSDirectory again and compare the two dumped string.
+ * </pre>
+ */
+ @Test
+ public void testSaveLoadImage() throws Exception {
+ // make changes to the namesystem
+ hdfs.mkdirs(dir);
+ hdfs.allowSnapshot(dir.toString());
+ hdfs.createSnapshot(dir, "s0");
+
+ Path sub1 = new Path(dir, "sub1");
+ Path sub1file1 = new Path(sub1, "sub1file1");
+ Path sub1file2 = new Path(sub1, "sub1file2");
+ DFSTestUtil.createFile(hdfs, sub1file1, BLOCKSIZE, REPLICATION, seed);
+ DFSTestUtil.createFile(hdfs, sub1file2, BLOCKSIZE, REPLICATION, seed);
+
+ hdfs.createSnapshot(dir, "s1");
+
+ Path sub2 = new Path(dir, "sub2");
+ Path sub2file1 = new Path(sub2, "sub2file1");
+ Path sub2file2 = new Path(sub2, "sub2file2");
+ DFSTestUtil.createFile(hdfs, sub2file1, BLOCKSIZE, REPLICATION, seed);
+ DFSTestUtil.createFile(hdfs, sub2file2, BLOCKSIZE, REPLICATION, seed);
+ hdfs.setReplication(sub1file1, (short) (REPLICATION - 1));
+ hdfs.delete(sub1file2, true);
+
+ hdfs.createSnapshot(dir, "s2");
+ hdfs.setOwner(sub2, "dr.who", "unknown");
+ hdfs.delete(sub2file2, true);
+
+ // dump the fsdir tree
+ StringBuffer fsnStrBefore = fsn.getFSDirectory().rootDir
+ .dumpTreeRecursively();
+
+ // save the namesystem to a temp file
+ SaveNamespaceContext context = new SaveNamespaceContext(fsn, txid,
+ new Canceler());
+ FSImageFormat.Saver saver = new FSImageFormat.Saver(context);
+ FSImageCompression compression = FSImageCompression.createCompression(conf);
+ File dstFile = getStorageFile(testDir, txid);
+ fsn.readLock();
+ try {
+ saver.save(dstFile, compression);
+ } finally {
+ fsn.readUnlock();
+ }
+
+ // restart the cluster, and format the cluster
+ cluster.shutdown();
+ cluster = new MiniDFSCluster.Builder(conf).format(true)
+ .numDataNodes(REPLICATION).build();
+ cluster.waitActive();
+ fsn = cluster.getNamesystem();
+ hdfs = cluster.getFileSystem();
+
+ // load the namesystem from the temp file
+ FSImageFormat.Loader loader = new FSImageFormat.Loader(conf, fsn);
+ fsn.writeLock();
+ try {
+ loader.load(dstFile);
+ } finally {
+ fsn.writeUnlock();
+ }
+
+ // dump the fsdir tree again
+ StringBuffer fsnStrAfter = fsn.getFSDirectory().rootDir
+ .dumpTreeRecursively();
+
+ // compare two dumped tree
+ System.out.println(fsnStrBefore.toString());
+ System.out.println("\n" + fsnStrAfter.toString());
+ assertEquals(fsnStrBefore.toString(), fsnStrAfter.toString());
+ }
+
+ /**
+ * Create a temp fsimage file for testing.
+ * @param dir The directory where the fsimage file resides
+ * @param imageTxId The transaction id of the fsimage
+ * @return The file of the image file
+ */
+ private File getStorageFile(String dir, long imageTxId) {
+ return new File(dir, String.format("%s_%019d", NameNodeFile.IMAGE,
+ imageTxId));
+ }
+}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java?rev=1437256&r1=1437255&r2=1437256&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java Wed Jan 23 02:48:01 2013
@@ -127,7 +127,6 @@ public class SnapshotTestHelper {
static class TestDirectoryTree {
/** Height of the directory tree */
final int height;
- final FileSystem fs;
/** Top node of the directory tree */
final Node topNode;
/** A map recording nodes for each tree level */
@@ -138,12 +137,11 @@ public class SnapshotTestHelper {
*/
TestDirectoryTree(int height, FileSystem fs) throws Exception {
this.height = height;
- this.fs = fs;
this.topNode = new Node(new Path("/TestSnapshot"), 0,
null, fs);
this.levelMap = new HashMap<Integer, ArrayList<Node>>();
addDirNode(topNode, 0);
- genChildren(topNode, height - 1);
+ genChildren(topNode, height - 1, fs);
}
/**
@@ -164,9 +162,11 @@ public class SnapshotTestHelper {
*
* @param parent The parent node
* @param level The remaining levels to generate
+ * @param fs The FileSystem where to generate the files/dirs
* @throws Exception
*/
- void genChildren(Node parent, int level) throws Exception {
+ private void genChildren(Node parent, int level, FileSystem fs)
+ throws Exception {
if (level == 0) {
return;
}
@@ -176,8 +176,8 @@ public class SnapshotTestHelper {
"right" + ++id), height - level, parent, fs);
addDirNode(parent.leftChild, parent.leftChild.level);
addDirNode(parent.rightChild, parent.rightChild.level);
- genChildren(parent.leftChild, level - 1);
- genChildren(parent.rightChild, level - 1);
+ genChildren(parent.leftChild, level - 1, fs);
+ genChildren(parent.rightChild, level - 1, fs);
}
/**
@@ -246,7 +246,6 @@ public class SnapshotTestHelper {
* directory creation/deletion
*/
final ArrayList<Node> nonSnapshotChildren;
- final FileSystem fs;
Node(Path path, int level, Node parent,
FileSystem fs) throws Exception {
@@ -254,7 +253,6 @@ public class SnapshotTestHelper {
this.level = level;
this.parent = parent;
this.nonSnapshotChildren = new ArrayList<Node>();
- this.fs = fs;
fs.mkdirs(nodePath);
}
@@ -262,8 +260,8 @@ public class SnapshotTestHelper {
* Create files and add them in the fileList. Initially the last element
* in the fileList is set to null (where we start file creation).
*/
- void initFileList(String namePrefix, long fileLen, short replication,
- long seed, int numFiles) throws Exception {
+ void initFileList(FileSystem fs, String namePrefix, long fileLen,
+ short replication, long seed, int numFiles) throws Exception {
fileList = new ArrayList<Path>(numFiles);
for (int i = 0; i < numFiles; i++) {
Path file = new Path(nodePath, namePrefix + "-f" + i);
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java?rev=1437256&r1=1437255&r2=1437256&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java Wed Jan 23 02:48:01 2013
@@ -148,7 +148,7 @@ public class TestINodeDirectoryWithSnaps
// combine all diffs
final Diff combined = diffs[0];
for(int i = 1; i < diffs.length; i++) {
- combined.combinePostDiff(diffs[i], null);
+ combined.combinePostDiff(diffs[i], null, false);
}
{
@@ -284,7 +284,7 @@ public class TestINodeDirectoryWithSnaps
before = toString(diff);
}
- final Triple<Integer, INode, Integer> undoInfo = diff.delete(inode);
+ final Triple<Integer, INode, Integer> undoInfo = diff.delete(inode, true);
if (testUndo) {
final String after = toString(diff);
@@ -292,7 +292,7 @@ public class TestINodeDirectoryWithSnaps
diff.undoDelete(inode, undoInfo);
assertDiff(before, diff);
//re-do
- diff.delete(inode);
+ diff.delete(inode, true);
assertDiff(after, diff);
}
}
@@ -314,7 +314,7 @@ public class TestINodeDirectoryWithSnaps
before = toString(diff);
}
- final Triple<Integer, INode, Integer> undoInfo = diff.modify(oldinode, newinode);
+ final Triple<Integer, INode, Integer> undoInfo = diff.modify(oldinode, newinode, true);
if (testUndo) {
final String after = toString(diff);
@@ -322,7 +322,7 @@ public class TestINodeDirectoryWithSnaps
diff.undoModify(oldinode, newinode, undoInfo);
assertDiff(before, diff);
//re-do
- diff.modify(oldinode, newinode);
+ diff.modify(oldinode, newinode, true);
assertDiff(after, diff);
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java?rev=1437256&r1=1437255&r2=1437256&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java Wed Jan 23 02:48:01 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper.TestDirectoryTree;
import org.apache.hadoop.ipc.RemoteException;
@@ -55,7 +56,7 @@ public class TestSnapshot {
private static final long seed = Time.now();
protected static final short REPLICATION = 3;
protected static final long BLOCKSIZE = 1024;
- /** The number of times snapshots are created for a snapshottable directory */
+ /** The number of times snapshots are created for a snapshottable directory */
public static final int SNAPSHOT_ITERATION_NUMBER = 20;
/** Height of directory tree used for testing */
public static final int DIRECTORY_TREE_LEVEL = 5;
@@ -143,6 +144,49 @@ public class TestSnapshot {
return nodes;
}
+ /**
+ * Restart the cluster to check edit log applying and fsimage saving/loading
+ */
+ private void checkFSImage() throws Exception {
+ String rootDir = "/";
+ StringBuffer fsnStrBefore = fsn.getFSDirectory().getINode(rootDir)
+ .dumpTreeRecursively();
+
+ cluster.shutdown();
+ cluster = new MiniDFSCluster.Builder(conf).format(false)
+ .numDataNodes(REPLICATION).build();
+ cluster.waitActive();
+ fsn = cluster.getNamesystem();
+ hdfs = cluster.getFileSystem();
+ // later check fsnStrMiddle to see if the edit log is recorded and applied
+ // correctly
+ StringBuffer fsnStrMiddle = fsn.getFSDirectory().getINode(rootDir)
+ .dumpTreeRecursively();
+
+ // save namespace and restart cluster
+ hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ hdfs.saveNamespace();
+ hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+ cluster.shutdown();
+ cluster = new MiniDFSCluster.Builder(conf).format(false)
+ .numDataNodes(REPLICATION).build();
+ cluster.waitActive();
+ fsn = cluster.getNamesystem();
+ hdfs = cluster.getFileSystem();
+ // dump the namespace loaded from fsimage
+ StringBuffer fsnStrAfter = fsn.getFSDirectory().getINode(rootDir)
+ .dumpTreeRecursively();
+
+ System.out.println("================== Original FSDir ==================");
+ System.out.println(fsnStrBefore.toString());
+ System.out.println("================== FSDir After Applying Edit Logs ==================");
+ System.out.println(fsnStrMiddle.toString());
+ System.out.println("================ FSDir After FSImage Saving/Loading ================");
+ System.out.println(fsnStrAfter.toString());
+ System.out.println("====================================================");
+ assertEquals(fsnStrBefore.toString(), fsnStrMiddle.toString());
+ assertEquals(fsnStrBefore.toString(), fsnStrAfter.toString());
+ }
/**
* Main test, where we will go in the following loop:
@@ -184,6 +228,9 @@ public class TestSnapshot {
Modification chown = new FileChown(chownDir.nodePath, hdfs, userGroup[0],
userGroup[1]);
modifyCurrentDirAndCheckSnapshots(new Modification[]{chmod, chown});
+
+ // check fsimage saving/loading
+ checkFSImage();
}
System.out.println("XXX done:");
SnapshotTestHelper.dumpTreeRecursively(fsn.getFSDirectory().getINode("/"));
@@ -244,7 +291,8 @@ public class TestSnapshot {
for (TestDirectoryTree.Node node : nodes) {
// If the node does not have files in it, create files
if (node.fileList == null) {
- node.initFileList(node.nodePath.getName(), BLOCKSIZE, REPLICATION, seed, 6);
+ node.initFileList(hdfs, node.nodePath.getName(), BLOCKSIZE,
+ REPLICATION, seed, 5);
}
//
@@ -270,18 +318,21 @@ public class TestSnapshot {
Modification delete = new FileDeletion(
node.fileList.get((node.nullFileIndex + 1) % node.fileList.size()),
hdfs);
- Modification append = new FileAppend(
- node.fileList.get((node.nullFileIndex + 2) % node.fileList.size()),
- hdfs, (int) BLOCKSIZE);
+
+ // TODO: temporarily disable file append testing before supporting
+ // INodeFileUnderConstructionWithSnapshot in FSImage saving/loading
+// Modification append = new FileAppend(
+// node.fileList.get((node.nullFileIndex + 2) % node.fileList.size()),
+// hdfs, (int) BLOCKSIZE);
Modification chmod = new FileChangePermission(
- node.fileList.get((node.nullFileIndex + 3) % node.fileList.size()),
+ node.fileList.get((node.nullFileIndex + 2) % node.fileList.size()),
hdfs, genRandomPermission());
String[] userGroup = genRandomOwner();
Modification chown = new FileChown(
- node.fileList.get((node.nullFileIndex + 4) % node.fileList.size()),
+ node.fileList.get((node.nullFileIndex + 3) % node.fileList.size()),
hdfs, userGroup[0], userGroup[1]);
Modification replication = new FileChangeReplication(
- node.fileList.get((node.nullFileIndex + 5) % node.fileList.size()),
+ node.fileList.get((node.nullFileIndex + 4) % node.fileList.size()),
hdfs, (short) (random.nextInt(REPLICATION) + 1));
node.nullFileIndex = (node.nullFileIndex + 1) % node.fileList.size();
Modification dirChange = new DirCreationOrDeletion(node.nodePath, hdfs,
@@ -289,7 +340,8 @@ public class TestSnapshot {
mList.add(create);
mList.add(delete);
- mList.add(append);
+ //TODO
+ //mList.add(append);
mList.add(chmod);
mList.add(chown);
mList.add(replication);
@@ -606,7 +658,7 @@ public class TestSnapshot {
/**
* Directory creation or deletion.
*/
- static class DirCreationOrDeletion extends Modification {
+ class DirCreationOrDeletion extends Modification {
private final TestDirectoryTree.Node node;
private final boolean isCreation;
private final Path changedPath;
@@ -656,15 +708,16 @@ public class TestSnapshot {
if (isCreation) {
// creation
TestDirectoryTree.Node newChild = new TestDirectoryTree.Node(
- changedPath, node.level + 1, node, node.fs);
+ changedPath, node.level + 1, node, hdfs);
// create file under the new non-snapshottable directory
- newChild.initFileList(node.nodePath.getName(), BLOCKSIZE, REPLICATION, seed, 2);
+ newChild.initFileList(hdfs, node.nodePath.getName(), BLOCKSIZE,
+ REPLICATION, seed, 2);
node.nonSnapshotChildren.add(newChild);
} else {
// deletion
TestDirectoryTree.Node childToDelete = node.nonSnapshotChildren
.remove(node.nonSnapshotChildren.size() - 1);
- node.fs.delete(childToDelete.nodePath, true);
+ hdfs.delete(childToDelete.nodePath, true);
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java?rev=1437256&r1=1437255&r2=1437256&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java Wed Jan 23 02:48:01 2013
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot.SnapshotDiff;
+import org.apache.hadoop.hdfs.util.ReadOnlyList;
import org.apache.hadoop.ipc.RemoteException;
import org.junit.After;
import org.junit.Before;
@@ -85,7 +86,7 @@ public class TestSnapshotRename {
*/
private void checkSnapshotList(INodeDirectorySnapshottable srcRoot,
String[] sortedNames, String[] names) {
- List<Snapshot> listByName = srcRoot.getSnapshotsByNames();
+ ReadOnlyList<Snapshot> listByName = srcRoot.getSnapshotsByNames();
assertEquals(sortedNames.length, listByName.size());
for (int i = 0; i < listByName.size(); i++) {
assertEquals(sortedNames[i], listByName.get(i).getRoot().getLocalName());