You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by um...@apache.org on 2014/05/15 15:03:26 UTC
svn commit: r1594906 [2/4] - in
/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs: ./ src/
src/main/bin/ src/main/java/ src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/protocol/
src/main/java/org/apache/hadoop...
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java Thu May 15 13:03:22 2014
@@ -21,14 +21,20 @@ import static org.apache.hadoop.util.Tim
import java.io.DataInput;
import java.io.DataInputStream;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.security.DigestInputStream;
+import java.security.DigestOutputStream;
import java.security.MessageDigest;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.TreeMap;
@@ -50,6 +56,7 @@ import org.apache.hadoop.hdfs.server.blo
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
@@ -60,6 +67,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType;
+import org.apache.hadoop.hdfs.util.ReadOnlyList;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.Text;
@@ -69,8 +77,105 @@ import com.google.common.base.Preconditi
import com.google.common.annotations.VisibleForTesting;
/**
- * This class loads and stores the FSImage of the NameNode. The file
- * src/main/proto/fsimage.proto describes the on-disk layout of the FSImage.
+ * Contains inner classes for reading or writing the on-disk format for
+ * FSImages.
+ *
+ * In particular, the format of the FSImage looks like:
+ * <pre>
+ * FSImage {
+ * layoutVersion: int, namespaceID: int, numberItemsInFSDirectoryTree: long,
+ * namesystemGenerationStampV1: long, namesystemGenerationStampV2: long,
+ * generationStampAtBlockIdSwitch:long, lastAllocatedBlockId:
+ * long transactionID: long, snapshotCounter: int, numberOfSnapshots: int,
+ * numOfSnapshottableDirs: int,
+ * {FSDirectoryTree, FilesUnderConstruction, SecretManagerState} (can be compressed)
+ * }
+ *
+ * FSDirectoryTree (if {@link Feature#FSIMAGE_NAME_OPTIMIZATION} is supported) {
+ * INodeInfo of root, numberOfChildren of root: int
+ * [list of INodeInfo of root's children],
+ * [list of INodeDirectoryInfo of root's directory children]
+ * }
+ *
+ * FSDirectoryTree (if {@link Feature#FSIMAGE_NAME_OPTIMIZATION} not supported){
+ * [list of INodeInfo of INodes in topological order]
+ * }
+ *
+ * INodeInfo {
+ * {
+ * localName: short + byte[]
+ * } when {@link Feature#FSIMAGE_NAME_OPTIMIZATION} is supported
+ * or
+ * {
+ * fullPath: byte[]
+ * } when {@link Feature#FSIMAGE_NAME_OPTIMIZATION} is not supported
+ * replicationFactor: short, modificationTime: long,
+ * accessTime: long, preferredBlockSize: long,
+ * numberOfBlocks: int (-1 for INodeDirectory, -2 for INodeSymLink),
+ * {
+ * nsQuota: long, dsQuota: long,
+ * {
+ * isINodeSnapshottable: byte,
+ * isINodeWithSnapshot: byte (if isINodeSnapshottable is false)
+ * } (when {@link Feature#SNAPSHOT} is supported),
+ * fsPermission: short, PermissionStatus
+ * } for INodeDirectory
+ * or
+ * {
+ * symlinkString, fsPermission: short, PermissionStatus
+ * } for INodeSymlink
+ * or
+ * {
+ * [list of BlockInfo]
+ * [list of FileDiff]
+ * {
+ * isINodeFileUnderConstructionSnapshot: byte,
+ * {clientName: short + byte[], clientMachine: short + byte[]} (when
+ * isINodeFileUnderConstructionSnapshot is true),
+ * } (when {@link Feature#SNAPSHOT} is supported and writing snapshotINode),
+ * fsPermission: short, PermissionStatus
+ * } for INodeFile
+ * }
+ *
+ * INodeDirectoryInfo {
+ * fullPath of the directory: short + byte[],
+ * numberOfChildren: int, [list of INodeInfo of children INode],
+ * {
+ * numberOfSnapshots: int,
+ * [list of Snapshot] (when NumberOfSnapshots is positive),
+ * numberOfDirectoryDiffs: int,
+ * [list of DirectoryDiff] (NumberOfDirectoryDiffs is positive),
+ * number of children that are directories,
+ * [list of INodeDirectoryInfo of the directory children] (includes
+ * snapshot copies of deleted sub-directories)
+ * } (when {@link Feature#SNAPSHOT} is supported),
+ * }
+ *
+ * Snapshot {
+ * snapshotID: int, root of Snapshot: INodeDirectoryInfo (its local name is
+ * the name of the snapshot)
+ * }
+ *
+ * DirectoryDiff {
+ * full path of the root of the associated Snapshot: short + byte[],
+ * childrenSize: int,
+ * isSnapshotRoot: byte,
+ * snapshotINodeIsNotNull: byte (when isSnapshotRoot is false),
+ * snapshotINode: INodeDirectory (when SnapshotINodeIsNotNull is true), Diff
+ * }
+ *
+ * Diff {
+ * createdListSize: int, [Local name of INode in created list],
+ * deletedListSize: int, [INode in deleted list: INodeInfo]
+ * }
+ *
+ * FileDiff {
+ * full path of the root of the associated Snapshot: short + byte[],
+ * fileSize: long,
+ * snapshotINodeIsNotNull: byte,
+ * snapshotINode: INodeFile (when SnapshotINodeIsNotNull is true), Diff
+ * }
+ * </pre>
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -580,6 +685,11 @@ public class FSImageFormat {
}
}
+ /** @return The FSDirectory of the namesystem where the fsimage is loaded */
+ public FSDirectory getFSDirectoryInLoading() {
+ return namesystem.dir;
+ }
+
public INode loadINodeWithLocalName(boolean isSnapshotINode, DataInput in,
boolean updateINodeMap) throws IOException {
return loadINodeWithLocalName(isSnapshotINode, in, updateINodeMap, null);
@@ -1009,7 +1119,7 @@ public class FSImageFormat {
+ " option to automatically rename these paths during upgrade.";
/**
- * Same as {@link #renameReservedPathsOnUpgrade}, but for a single
+ * Same as {@link #renameReservedPathsOnUpgrade(String)}, but for a single
* byte array path component.
*/
private static byte[] renameReservedComponentOnUpgrade(byte[] component,
@@ -1029,7 +1139,7 @@ public class FSImageFormat {
}
/**
- * Same as {@link #renameReservedPathsOnUpgrade}, but for a single
+ * Same as {@link #renameReservedPathsOnUpgrade(String)}, but for a single
* byte array path component.
*/
private static byte[] renameReservedRootComponentOnUpgrade(byte[] component,
@@ -1050,4 +1160,271 @@ public class FSImageFormat {
}
return component;
}
+
+ /**
+ * A one-shot class responsible for writing an image file.
+ * The write() function should be called once, after which the getter
+ * functions may be used to retrieve information about the file that was written.
+ *
+ * This is replaced by the PB-based FSImage. The class is to maintain
+ * compatibility for the external fsimage tool.
+ */
+ @Deprecated
+ static class Saver {
+ private static final int LAYOUT_VERSION = -51;
+ private final SaveNamespaceContext context;
+ /** Set to true once an image has been written */
+ private boolean saved = false;
+
+ /** The MD5 checksum of the file that was written */
+ private MD5Hash savedDigest;
+ private final ReferenceMap referenceMap = new ReferenceMap();
+
+ private final Map<Long, INodeFile> snapshotUCMap =
+ new HashMap<Long, INodeFile>();
+
+ /** @throws IllegalStateException if the instance has not yet saved an image */
+ private void checkSaved() {
+ if (!saved) {
+ throw new IllegalStateException("FSImageSaver has not saved an image");
+ }
+ }
+
+ /** @throws IllegalStateException if the instance has already saved an image */
+ private void checkNotSaved() {
+ if (saved) {
+ throw new IllegalStateException("FSImageSaver has already saved an image");
+ }
+ }
+
+
+ Saver(SaveNamespaceContext context) {
+ this.context = context;
+ }
+
+ /**
+ * Return the MD5 checksum of the image file that was saved.
+ */
+ MD5Hash getSavedDigest() {
+ checkSaved();
+ return savedDigest;
+ }
+
+ void save(File newFile, FSImageCompression compression) throws IOException {
+ checkNotSaved();
+
+ final FSNamesystem sourceNamesystem = context.getSourceNamesystem();
+ final INodeDirectory rootDir = sourceNamesystem.dir.rootDir;
+ final long numINodes = rootDir.getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().get(Quota.NAMESPACE);
+ String sdPath = newFile.getParentFile().getParentFile().getAbsolutePath();
+ Step step = new Step(StepType.INODES, sdPath);
+ StartupProgress prog = NameNode.getStartupProgress();
+ prog.beginStep(Phase.SAVING_CHECKPOINT, step);
+ prog.setTotal(Phase.SAVING_CHECKPOINT, step, numINodes);
+ Counter counter = prog.getCounter(Phase.SAVING_CHECKPOINT, step);
+ long startTime = now();
+ //
+ // Write out data
+ //
+ MessageDigest digester = MD5Hash.getDigester();
+ FileOutputStream fout = new FileOutputStream(newFile);
+ DigestOutputStream fos = new DigestOutputStream(fout, digester);
+ DataOutputStream out = new DataOutputStream(fos);
+ try {
+ out.writeInt(LAYOUT_VERSION);
+ LayoutFlags.write(out);
+ // We use the non-locked version of getNamespaceInfo here since
+ // the coordinating thread of saveNamespace already has read-locked
+ // the namespace for us. If we attempt to take another readlock
+ // from the actual saver thread, there's a potential of a
+ // fairness-related deadlock. See the comments on HDFS-2223.
+ out.writeInt(sourceNamesystem.unprotectedGetNamespaceInfo()
+ .getNamespaceID());
+ out.writeLong(numINodes);
+ out.writeLong(sourceNamesystem.getGenerationStampV1());
+ out.writeLong(sourceNamesystem.getGenerationStampV2());
+ out.writeLong(sourceNamesystem.getGenerationStampAtblockIdSwitch());
+ out.writeLong(sourceNamesystem.getLastAllocatedBlockId());
+ out.writeLong(context.getTxId());
+ out.writeLong(sourceNamesystem.getLastInodeId());
+
+
+ sourceNamesystem.getSnapshotManager().write(out);
+
+ // write compression info and set up compressed stream
+ out = compression.writeHeaderAndWrapStream(fos);
+ LOG.info("Saving image file " + newFile +
+ " using " + compression);
+
+ // save the root
+ saveINode2Image(rootDir, out, false, referenceMap, counter);
+ // save the rest of the nodes
+ saveImage(rootDir, out, true, false, counter);
+ prog.endStep(Phase.SAVING_CHECKPOINT, step);
+ // Now that the step is finished, set counter equal to total to adjust
+ // for possible under-counting due to reference inodes.
+ prog.setCount(Phase.SAVING_CHECKPOINT, step, numINodes);
+ // save files under construction
+ // TODO: for HDFS-5428, since we cannot break the compatibility of
+ // fsimage, we store part of the under-construction files that are only
+ // in snapshots in this "under-construction-file" section. As a
+ // temporary solution, we use "/.reserved/.inodes/<inodeid>" as their
+ // paths, so that when loading fsimage we do not put them into the lease
+ // map. In the future, we can remove this hack when we can bump the
+ // layout version.
+ sourceNamesystem.saveFilesUnderConstruction(out, snapshotUCMap);
+
+ context.checkCancelled();
+ sourceNamesystem.saveSecretManagerStateCompat(out, sdPath);
+ context.checkCancelled();
+ sourceNamesystem.getCacheManager().saveStateCompat(out, sdPath);
+ context.checkCancelled();
+ out.flush();
+ context.checkCancelled();
+ fout.getChannel().force(true);
+ } finally {
+ out.close();
+ }
+
+ saved = true;
+ // set md5 of the saved image
+ savedDigest = new MD5Hash(digester.digest());
+
+ LOG.info("Image file " + newFile + " of size " + newFile.length() +
+ " bytes saved in " + (now() - startTime)/1000 + " seconds.");
+ }
+
+ /**
+ * Save children INodes.
+ * @param children The list of children INodes
+ * @param out The DataOutputStream to write
+ * @param inSnapshot Whether the parent directory or its ancestor is in
+ * the deleted list of some snapshot (caused by rename or
+ * deletion)
+ * @param counter Counter to increment for namenode startup progress
+ * @return Number of children that are directory
+ */
+ private int saveChildren(ReadOnlyList<INode> children,
+ DataOutputStream out, boolean inSnapshot, Counter counter)
+ throws IOException {
+ // Write normal children INode.
+ out.writeInt(children.size());
+ int dirNum = 0;
+ int i = 0;
+ for(INode child : children) {
+ // print all children first
+ // TODO: for HDFS-5428, we cannot change the format/content of fsimage
+ // here, thus even if the parent directory is in snapshot, we still
+ // do not handle INodeUC as those stored in deleted list
+ saveINode2Image(child, out, false, referenceMap, counter);
+ if (child.isDirectory()) {
+ dirNum++;
+ } else if (inSnapshot && child.isFile()
+ && child.asFile().isUnderConstruction()) {
+ this.snapshotUCMap.put(child.getId(), child.asFile());
+ }
+ if (i++ % 50 == 0) {
+ context.checkCancelled();
+ }
+ }
+ return dirNum;
+ }
+
+ /**
+ * Save file tree image starting from the given root.
+ * This is a recursive procedure, which first saves all children and
+ * snapshot diffs of a current directory and then moves inside the
+ * sub-directories.
+ *
+ * @param current The current node
+ * @param out The DataoutputStream to write the image
+ * @param toSaveSubtree Whether or not to save the subtree to fsimage. For
+ * reference node, its subtree may already have been
+ * saved before.
+ * @param inSnapshot Whether the current directory is in snapshot
+ * @param counter Counter to increment for namenode startup progress
+ */
+ private void saveImage(INodeDirectory current, DataOutputStream out,
+ boolean toSaveSubtree, boolean inSnapshot, Counter counter)
+ throws IOException {
+ // write the inode id of the directory
+ out.writeLong(current.getId());
+
+ if (!toSaveSubtree) {
+ return;
+ }
+
+ final ReadOnlyList<INode> children = current
+ .getChildrenList(Snapshot.CURRENT_STATE_ID);
+ int dirNum = 0;
+ List<INodeDirectory> snapshotDirs = null;
+ DirectoryWithSnapshotFeature sf = current.getDirectoryWithSnapshotFeature();
+ if (sf != null) {
+ snapshotDirs = new ArrayList<INodeDirectory>();
+ sf.getSnapshotDirectory(snapshotDirs);
+ dirNum += snapshotDirs.size();
+ }
+
+ // 2. Write INodeDirectorySnapshottable#snapshotsByNames to record all
+ // Snapshots
+ if (current instanceof INodeDirectorySnapshottable) {
+ INodeDirectorySnapshottable snapshottableNode =
+ (INodeDirectorySnapshottable) current;
+ SnapshotFSImageFormat.saveSnapshots(snapshottableNode, out);
+ } else {
+ out.writeInt(-1); // # of snapshots
+ }
+
+ // 3. Write children INode
+ dirNum += saveChildren(children, out, inSnapshot, counter);
+
+ // 4. Write DirectoryDiff lists, if there is any.
+ SnapshotFSImageFormat.saveDirectoryDiffList(current, out, referenceMap);
+
+ // Write sub-tree of sub-directories, including possible snapshots of
+ // deleted sub-directories
+ out.writeInt(dirNum); // the number of sub-directories
+ for(INode child : children) {
+ if(!child.isDirectory()) {
+ continue;
+ }
+ // make sure we only save the subtree under a reference node once
+ boolean toSave = child.isReference() ?
+ referenceMap.toProcessSubtree(child.getId()) : true;
+ saveImage(child.asDirectory(), out, toSave, inSnapshot, counter);
+ }
+ if (snapshotDirs != null) {
+ for (INodeDirectory subDir : snapshotDirs) {
+ // make sure we only save the subtree under a reference node once
+ boolean toSave = subDir.getParentReference() != null ?
+ referenceMap.toProcessSubtree(subDir.getId()) : true;
+ saveImage(subDir, out, toSave, true, counter);
+ }
+ }
+ }
+
+ /**
+ * Saves inode and increments progress counter.
+ *
+ * @param inode INode to save
+ * @param out DataOutputStream to receive inode
+ * @param writeUnderConstruction boolean true if this is under construction
+ * @param referenceMap ReferenceMap containing reference inodes
+ * @param counter Counter to increment for namenode startup progress
+ * @throws IOException thrown if there is an I/O error
+ */
+ private void saveINode2Image(INode inode, DataOutputStream out,
+ boolean writeUnderConstruction, ReferenceMap referenceMap,
+ Counter counter) throws IOException {
+ FSImageSerialization.saveINode2Image(inode, out, writeUnderConstruction,
+ referenceMap);
+ // Intentionally do not increment counter for reference inodes, because it
+ // is too difficult at this point to assess whether or not this is a
+ // reference that counts toward quota.
+ if (!(inode instanceof INodeReference)) {
+ counter.increment();
+ }
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java Thu May 15 13:03:22 2014
@@ -17,6 +17,11 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
@@ -31,20 +36,21 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
import org.apache.hadoop.hdfs.util.XMLUtils;
import org.apache.hadoop.hdfs.util.XMLUtils.InvalidXmlException;
import org.apache.hadoop.hdfs.util.XMLUtils.Stanza;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.ShortWritable;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.IOException;
+import com.google.common.base.Preconditions;
/**
* Static utility functions for serializing various pieces of data in the correct
@@ -82,6 +88,26 @@ public class FSImageSerialization {
final ShortWritable U_SHORT = new ShortWritable();
final IntWritable U_INT = new IntWritable();
final LongWritable U_LONG = new LongWritable();
+ final FsPermission FILE_PERM = new FsPermission((short) 0);
+ }
+
+ private static void writePermissionStatus(INodeAttributes inode,
+ DataOutput out) throws IOException {
+ final FsPermission p = TL_DATA.get().FILE_PERM;
+ p.fromShort(inode.getFsPermissionShort());
+ PermissionStatus.write(out, inode.getUserName(), inode.getGroupName(), p);
+ }
+
+ private static void writeBlocks(final Block[] blocks,
+ final DataOutput out) throws IOException {
+ if (blocks == null) {
+ out.writeInt(0);
+ } else {
+ out.writeInt(blocks.length);
+ for (Block blk : blocks) {
+ blk.write(out);
+ }
+ }
}
// Helper function that reads in an INodeUnderConstruction
@@ -127,6 +153,183 @@ public class FSImageSerialization {
return file;
}
+ // Helper function that writes an INodeUnderConstruction
+ // into the input stream
+ //
+ static void writeINodeUnderConstruction(DataOutputStream out, INodeFile cons,
+ String path) throws IOException {
+ writeString(path, out);
+ out.writeLong(cons.getId());
+ out.writeShort(cons.getFileReplication());
+ out.writeLong(cons.getModificationTime());
+ out.writeLong(cons.getPreferredBlockSize());
+
+ writeBlocks(cons.getBlocks(), out);
+ cons.getPermissionStatus().write(out);
+
+ FileUnderConstructionFeature uc = cons.getFileUnderConstructionFeature();
+ writeString(uc.getClientName(), out);
+ writeString(uc.getClientMachine(), out);
+
+ out.writeInt(0); // do not store locations of last block
+ }
+
+ /**
+ * Serialize a {@link INodeFile} node
+ * @param node The node to write
+ * @param out The {@link DataOutputStream} where the fields are written
+ * @param writeBlock Whether to write block information
+ */
+ public static void writeINodeFile(INodeFile file, DataOutput out,
+ boolean writeUnderConstruction) throws IOException {
+ writeLocalName(file, out);
+ out.writeLong(file.getId());
+ out.writeShort(file.getFileReplication());
+ out.writeLong(file.getModificationTime());
+ out.writeLong(file.getAccessTime());
+ out.writeLong(file.getPreferredBlockSize());
+
+ writeBlocks(file.getBlocks(), out);
+ SnapshotFSImageFormat.saveFileDiffList(file, out);
+
+ if (writeUnderConstruction) {
+ if (file.isUnderConstruction()) {
+ out.writeBoolean(true);
+ final FileUnderConstructionFeature uc = file.getFileUnderConstructionFeature();
+ writeString(uc.getClientName(), out);
+ writeString(uc.getClientMachine(), out);
+ } else {
+ out.writeBoolean(false);
+ }
+ }
+
+ writePermissionStatus(file, out);
+ }
+
+ /** Serialize an {@link INodeFileAttributes}. */
+ public static void writeINodeFileAttributes(INodeFileAttributes file,
+ DataOutput out) throws IOException {
+ writeLocalName(file, out);
+ writePermissionStatus(file, out);
+ out.writeLong(file.getModificationTime());
+ out.writeLong(file.getAccessTime());
+
+ out.writeShort(file.getFileReplication());
+ out.writeLong(file.getPreferredBlockSize());
+ }
+
+ private static void writeQuota(Quota.Counts quota, DataOutput out)
+ throws IOException {
+ out.writeLong(quota.get(Quota.NAMESPACE));
+ out.writeLong(quota.get(Quota.DISKSPACE));
+ }
+
+ /**
+ * Serialize a {@link INodeDirectory}
+ * @param node The node to write
+ * @param out The {@link DataOutput} where the fields are written
+ */
+ public static void writeINodeDirectory(INodeDirectory node, DataOutput out)
+ throws IOException {
+ writeLocalName(node, out);
+ out.writeLong(node.getId());
+ out.writeShort(0); // replication
+ out.writeLong(node.getModificationTime());
+ out.writeLong(0); // access time
+ out.writeLong(0); // preferred block size
+ out.writeInt(-1); // # of blocks
+
+ writeQuota(node.getQuotaCounts(), out);
+
+ if (node instanceof INodeDirectorySnapshottable) {
+ out.writeBoolean(true);
+ } else {
+ out.writeBoolean(false);
+ out.writeBoolean(node.isWithSnapshot());
+ }
+
+ writePermissionStatus(node, out);
+ }
+
+ /**
+ * Serialize a {@link INodeDirectory}
+ * @param a The node to write
+ * @param out The {@link DataOutput} where the fields are written
+ */
+ public static void writeINodeDirectoryAttributes(
+ INodeDirectoryAttributes a, DataOutput out) throws IOException {
+ writeLocalName(a, out);
+ writePermissionStatus(a, out);
+ out.writeLong(a.getModificationTime());
+ writeQuota(a.getQuotaCounts(), out);
+ }
+
+ /**
+ * Serialize a {@link INodeSymlink} node
+ * @param node The node to write
+ * @param out The {@link DataOutput} where the fields are written
+ */
+ private static void writeINodeSymlink(INodeSymlink node, DataOutput out)
+ throws IOException {
+ writeLocalName(node, out);
+ out.writeLong(node.getId());
+ out.writeShort(0); // replication
+ out.writeLong(0); // modification time
+ out.writeLong(0); // access time
+ out.writeLong(0); // preferred block size
+ out.writeInt(-2); // # of blocks
+
+ Text.writeString(out, node.getSymlinkString());
+ writePermissionStatus(node, out);
+ }
+
+ /** Serialize a {@link INodeReference} node */
+ private static void writeINodeReference(INodeReference ref, DataOutput out,
+ boolean writeUnderConstruction, ReferenceMap referenceMap
+ ) throws IOException {
+ writeLocalName(ref, out);
+ out.writeLong(ref.getId());
+ out.writeShort(0); // replication
+ out.writeLong(0); // modification time
+ out.writeLong(0); // access time
+ out.writeLong(0); // preferred block size
+ out.writeInt(-3); // # of blocks
+
+ final boolean isWithName = ref instanceof INodeReference.WithName;
+ out.writeBoolean(isWithName);
+
+ if (!isWithName) {
+ Preconditions.checkState(ref instanceof INodeReference.DstReference);
+ // dst snapshot id
+ out.writeInt(((INodeReference.DstReference) ref).getDstSnapshotId());
+ } else {
+ out.writeInt(((INodeReference.WithName) ref).getLastSnapshotId());
+ }
+
+ final INodeReference.WithCount withCount
+ = (INodeReference.WithCount)ref.getReferredINode();
+ referenceMap.writeINodeReferenceWithCount(withCount, out,
+ writeUnderConstruction);
+ }
+
+ /**
+ * Save one inode's attributes to the image.
+ */
+ public static void saveINode2Image(INode node, DataOutput out,
+ boolean writeUnderConstruction, ReferenceMap referenceMap)
+ throws IOException {
+ if (node.isReference()) {
+ writeINodeReference(node.asReference(), out, writeUnderConstruction,
+ referenceMap);
+ } else if (node.isDirectory()) {
+ writeINodeDirectory(node.asDirectory(), out);
+ } else if (node.isSymlink()) {
+ writeINodeSymlink(node.asSymlink(), out);
+ } else if (node.isFile()) {
+ writeINodeFile(node.asFile(), out, writeUnderConstruction);
+ }
+ }
+
// This should be reverted to package private once the ImageLoader
// code is moved into this package. This method should not be called
// by other code.
@@ -226,6 +429,12 @@ public class FSImageSerialization {
in.readFully(createdNodeName);
return createdNodeName;
}
+
+ private static void writeLocalName(INodeAttributes inode, DataOutput out)
+ throws IOException {
+ final byte[] name = inode.getLocalNameBytes();
+ writeBytes(name, out);
+ }
public static void writeBytes(byte[] data, DataOutput out)
throws IOException {
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Thu May 15 13:03:22 2014
@@ -85,17 +85,7 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
import static org.apache.hadoop.util.Time.now;
-import java.io.BufferedWriter;
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.io.StringWriter;
+import java.io.*;
import java.io.UnsupportedEncodingException;
import java.lang.management.ManagementFactory;
import java.net.InetAddress;
@@ -140,6 +130,7 @@ import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
@@ -3248,10 +3239,11 @@ public class FSNamesystem implements Nam
// Rename does not operates on link targets
// Do not resolveLink when checking permissions of src and dst
// Check write access to parent of src
- checkPermission(pc, src, false, null, FsAction.WRITE, null, null, false);
+ checkPermission(pc, src, false, null, FsAction.WRITE, null, null,
+ false, false);
// Check write access to ancestor of dst
checkPermission(pc, actualdst, false, FsAction.WRITE, null, null, null,
- false);
+ false, false);
}
if (dir.renameTo(src, dst, logRetryCache)) {
@@ -3312,9 +3304,11 @@ public class FSNamesystem implements Nam
// Rename does not operates on link targets
// Do not resolveLink when checking permissions of src and dst
// Check write access to parent of src
- checkPermission(pc, src, false, null, FsAction.WRITE, null, null, false);
+ checkPermission(pc, src, false, null, FsAction.WRITE, null, null, false,
+ false);
// Check write access to ancestor of dst
- checkPermission(pc, dst, false, FsAction.WRITE, null, null, null, false);
+ checkPermission(pc, dst, false, FsAction.WRITE, null, null, null, false,
+ false);
}
dir.renameTo(src, dst, logRetryCache, options);
@@ -3394,11 +3388,11 @@ public class FSNamesystem implements Nam
checkNameNodeSafeMode("Cannot delete " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
if (!recursive && dir.isNonEmptyDirectory(src)) {
- throw new IOException(src + " is non empty");
+ throw new PathIsNotEmptyDirectoryException(src + " is non empty");
}
if (enforcePermission && isPermissionEnabled) {
checkPermission(pc, src, false, null, FsAction.WRITE, null,
- FsAction.ALL, false);
+ FsAction.ALL, true, false);
}
// Unlink the target directory from directory tree
if (!dir.delete(src, collectedBlocks, removedINodes, logRetryCache)) {
@@ -3550,7 +3544,8 @@ public class FSNamesystem implements Nam
checkOperation(OperationCategory.READ);
src = FSDirectory.resolvePath(src, pathComponents, dir);
if (isPermissionEnabled) {
- checkPermission(pc, src, false, null, null, null, null, resolveLink);
+ checkPermission(pc, src, false, null, null, null, null, false,
+ resolveLink);
}
stat = dir.getFileInfo(src, resolveLink);
} catch (AccessControlException e) {
@@ -5549,7 +5544,7 @@ public class FSNamesystem implements Nam
FsAction parentAccess, FsAction access, FsAction subAccess)
throws AccessControlException, UnresolvedLinkException {
checkPermission(pc, path, doCheckOwner, ancestorAccess,
- parentAccess, access, subAccess, true);
+ parentAccess, access, subAccess, false, true);
}
/**
@@ -5560,14 +5555,14 @@ public class FSNamesystem implements Nam
private void checkPermission(FSPermissionChecker pc,
String path, boolean doCheckOwner, FsAction ancestorAccess,
FsAction parentAccess, FsAction access, FsAction subAccess,
- boolean resolveLink)
+ boolean ignoreEmptyDir, boolean resolveLink)
throws AccessControlException, UnresolvedLinkException {
if (!pc.isSuperUser()) {
dir.waitForReady();
readLock();
try {
pc.checkPermission(path, dir, doCheckOwner, ancestorAccess,
- parentAccess, access, subAccess, resolveLink);
+ parentAccess, access, subAccess, ignoreEmptyDir, resolveLink);
} finally {
readUnlock();
}
@@ -6092,6 +6087,42 @@ public class FSNamesystem implements Nam
}
/**
+ * Serializes leases.
+ */
+ void saveFilesUnderConstruction(DataOutputStream out,
+ Map<Long, INodeFile> snapshotUCMap) throws IOException {
+ // This is run by an inferior thread of saveNamespace, which holds a read
+ // lock on our behalf. If we took the read lock here, we could block
+ // for fairness if a writer is waiting on the lock.
+ synchronized (leaseManager) {
+ Map<String, INodeFile> nodes = leaseManager.getINodesUnderConstruction();
+ for (Map.Entry<String, INodeFile> entry : nodes.entrySet()) {
+ // TODO: for HDFS-5428, because of rename operations, some
+ // under-construction files that are
+ // in the current fs directory can also be captured in the
+ // snapshotUCMap. We should remove them from the snapshotUCMap.
+ snapshotUCMap.remove(entry.getValue().getId());
+ }
+
+ out.writeInt(nodes.size() + snapshotUCMap.size()); // write the size
+ for (Map.Entry<String, INodeFile> entry : nodes.entrySet()) {
+ FSImageSerialization.writeINodeUnderConstruction(
+ out, entry.getValue(), entry.getKey());
+ }
+ for (Map.Entry<Long, INodeFile> entry : snapshotUCMap.entrySet()) {
+ // for those snapshot INodeFileUC, we use "/.reserved/.inodes/<inodeid>"
+ // as their paths
+ StringBuilder b = new StringBuilder();
+ b.append(FSDirectory.DOT_RESERVED_PATH_PREFIX)
+ .append(Path.SEPARATOR).append(FSDirectory.DOT_INODES_STRING)
+ .append(Path.SEPARATOR).append(entry.getValue().getId());
+ FSImageSerialization.writeINodeUnderConstruction(
+ out, entry.getValue(), b.toString());
+ }
+ }
+ }
+
+ /**
* @return all the under-construction files in the lease map
*/
Map<String, INodeFile> getFilesUnderConstruction() {
@@ -6377,6 +6408,15 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
}
+ /**
+ * @param out save state of the secret manager
+ * @param sdPath String storage directory path
+ */
+ void saveSecretManagerStateCompat(DataOutputStream out, String sdPath)
+ throws IOException {
+ dtSecretManager.saveSecretManagerStateCompat(out, sdPath);
+ }
+
SecretManagerState saveSecretManagerState() {
return dtSecretManager.saveSecretManagerState();
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java Thu May 15 13:03:22 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.permission.A
import org.apache.hadoop.fs.permission.AclEntryType;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.util.ReadOnlyList;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
@@ -136,6 +137,7 @@ class FSPermissionChecker {
* @param subAccess If path is a directory,
* it is the access required of the path and all the sub-directories.
* If path is not a directory, there is no effect.
+ * @param ignoreEmptyDir Ignore permission checking for empty directory?
* @param resolveLink whether to resolve the final path component if it is
* a symlink
* @throws AccessControlException
@@ -146,7 +148,7 @@ class FSPermissionChecker {
*/
void checkPermission(String path, FSDirectory dir, boolean doCheckOwner,
FsAction ancestorAccess, FsAction parentAccess, FsAction access,
- FsAction subAccess, boolean resolveLink)
+ FsAction subAccess, boolean ignoreEmptyDir, boolean resolveLink)
throws AccessControlException, UnresolvedLinkException {
if (LOG.isDebugEnabled()) {
LOG.debug("ACCESS CHECK: " + this
@@ -155,6 +157,7 @@ class FSPermissionChecker {
+ ", parentAccess=" + parentAccess
+ ", access=" + access
+ ", subAccess=" + subAccess
+ + ", ignoreEmptyDir=" + ignoreEmptyDir
+ ", resolveLink=" + resolveLink);
}
// check if (parentAccess != null) && file exists, then check sb
@@ -182,7 +185,7 @@ class FSPermissionChecker {
check(last, snapshotId, access);
}
if (subAccess != null) {
- checkSubAccess(last, snapshotId, subAccess);
+ checkSubAccess(last, snapshotId, subAccess, ignoreEmptyDir);
}
if (doCheckOwner) {
checkOwner(last, snapshotId);
@@ -207,8 +210,8 @@ class FSPermissionChecker {
}
/** Guarded by {@link FSNamesystem#readLock()} */
- private void checkSubAccess(INode inode, int snapshotId, FsAction access
- ) throws AccessControlException {
+ private void checkSubAccess(INode inode, int snapshotId, FsAction access,
+ boolean ignoreEmptyDir) throws AccessControlException {
if (inode == null || !inode.isDirectory()) {
return;
}
@@ -216,9 +219,12 @@ class FSPermissionChecker {
Stack<INodeDirectory> directories = new Stack<INodeDirectory>();
for(directories.push(inode.asDirectory()); !directories.isEmpty(); ) {
INodeDirectory d = directories.pop();
- check(d, snapshotId, access);
+ ReadOnlyList<INode> cList = d.getChildrenList(snapshotId);
+ if (!(cList.isEmpty() && ignoreEmptyDir)) {
+ check(d, snapshotId, access);
+ }
- for(INode child : d.getChildrenList(snapshotId)) {
+ for(INode child : cList) {
if (child.isDirectory()) {
directories.push(child.asDirectory());
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java Thu May 15 13:03:22 2014
@@ -40,7 +40,7 @@ import com.google.common.base.Preconditi
* snapshots and it is renamed/moved to other locations.
*
* For example,
- * (1) Support we have /abc/foo, say the inode of foo is inode(id=1000,name=foo)
+ * (1) Suppose we have /abc/foo, say the inode of foo is inode(id=1000,name=foo)
* (2) create snapshot s0 for /abc
* (3) mv /abc/foo /xyz/bar, i.e. inode(id=1000,name=...) is renamed from "foo"
* to "bar" and its parent becomes /xyz.
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Thu May 15 13:03:22 2014
@@ -77,7 +77,8 @@ public class NNStorage extends Storage i
IMAGE_ROLLBACK("fsimage_rollback"),
EDITS_NEW ("edits.new"), // from "old" pre-HDFS-1073 format
EDITS_INPROGRESS ("edits_inprogress"),
- EDITS_TMP ("edits_tmp");
+ EDITS_TMP ("edits_tmp"),
+ IMAGE_LEGACY_OIV ("fsimage_legacy_oiv"); // For pre-PB format
private String fileName = null;
private NameNodeFile(String name) { this.fileName = name; }
@@ -693,6 +694,10 @@ public class NNStorage extends Storage i
return getNameNodeFileName(NameNodeFile.IMAGE_ROLLBACK, txid);
}
+ public static String getLegacyOIVImageFileName(long txid) {
+ return getNameNodeFileName(NameNodeFile.IMAGE_LEGACY_OIV, txid);
+ }
+
private static String getNameNodeFileName(NameNodeFile nnf, long txid) {
return String.format("%s_%019d", nnf.getName(), txid);
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java Thu May 15 13:03:22 2014
@@ -18,11 +18,13 @@
package org.apache.hadoop.hdfs.server.namenode;
import java.io.File;
+import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumSet;
+import java.util.Iterator;
import java.util.List;
import java.util.TreeSet;
@@ -233,4 +235,58 @@ public class NNStorageRetentionManager {
}
}
}
+
+ /**
+ * Delete old OIV fsimages. Since the target dir is not a full blown
+ * storage directory, we simply list and keep the latest ones. For the
+ * same reason, no storage inspector is used.
+ */
+ void purgeOldLegacyOIVImages(String dir, long txid) {
+ File oivImageDir = new File(dir);
+ final String oivImagePrefix = NameNodeFile.IMAGE_LEGACY_OIV.getName();
+ String filesInStorage[];
+
+ // Get the listing
+ filesInStorage = oivImageDir.list(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.matches(oivImagePrefix + "_(\\d+)");
+ }
+ });
+
+ // Check whether there is any work to do.
+ if (filesInStorage.length <= numCheckpointsToRetain) {
+ return;
+ }
+
+ // Create a sorted list of txids from the file names.
+ TreeSet<Long> sortedTxIds = new TreeSet<Long>();
+ for (String fName : filesInStorage) {
+ // Extract the transaction id from the file name.
+ long fTxId;
+ try {
+ fTxId = Long.parseLong(fName.substring(oivImagePrefix.length() + 1));
+ } catch (NumberFormatException nfe) {
+ // This should not happen since we have already filtered it.
+ // Log and continue.
+ LOG.warn("Invalid file name. Skipping " + fName);
+ continue;
+ }
+ sortedTxIds.add(Long.valueOf(fTxId));
+ }
+
+ int numFilesToDelete = sortedTxIds.size() - numCheckpointsToRetain;
+ Iterator<Long> iter = sortedTxIds.iterator();
+ while (numFilesToDelete > 0 && iter.hasNext()) {
+ long txIdVal = iter.next().longValue();
+ String fileName = NNStorage.getLegacyOIVImageFileName(txIdVal);
+ LOG.info("Deleting " + fileName);
+ File fileToDelete = new File(oivImageDir, fileName);
+ if (!fileToDelete.delete()) {
+ // deletion failed.
+ LOG.warn("Failed to delete image file: " + fileToDelete);
+ }
+ numFilesToDelete--;
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Thu May 15 13:03:22 2014
@@ -1677,7 +1677,11 @@ public class NameNode implements NameNod
public boolean isStandbyState() {
return (state.equals(STANDBY_STATE));
}
-
+
+ public boolean isActiveState() {
+ return (state.equals(ACTIVE_STATE));
+ }
+
/**
* Check that a request to change this node's HA state is valid.
* In particular, verifies that, if auto failover is enabled, non-forced
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Thu May 15 13:03:22 2014
@@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
+import org.apache.hadoop.hdfs.util.Canceler;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.io.MD5Hash;
@@ -125,6 +126,7 @@ public class SecondaryNameNode implement
private Thread checkpointThread;
private ObjectName nameNodeStatusBeanName;
+ private String legacyOivImageDir;
@Override
public String toString() {
@@ -289,6 +291,9 @@ public class SecondaryNameNode implement
NetUtils.getHostPortString(httpsAddress));
}
+ legacyOivImageDir = conf.get(
+ DFSConfigKeys.DFS_NAMENODE_LEGACY_OIV_IMAGE_DIR_KEY);
+
LOG.info("Checkpoint Period :" + checkpointConf.getPeriod() + " secs "
+ "(" + checkpointConf.getPeriod() / 60 + " min)");
LOG.info("Log Size Trigger :" + checkpointConf.getTxnCount() + " txns");
@@ -497,6 +502,7 @@ public class SecondaryNameNode implement
* @return if the image is fetched from primary or not
*/
@VisibleForTesting
+ @SuppressWarnings("deprecated")
public boolean doCheckpoint() throws IOException {
checkpointImage.ensureCurrentDirExists();
NNStorage dstStorage = checkpointImage.getStorage();
@@ -559,11 +565,18 @@ public class SecondaryNameNode implement
LOG.warn("Checkpoint done. New Image Size: "
+ dstStorage.getFsImageName(txid).length());
-
+
+ if (legacyOivImageDir != null && !legacyOivImageDir.isEmpty()) {
+ try {
+ checkpointImage.saveLegacyOIVImage(namesystem, legacyOivImageDir,
+ new Canceler());
+ } catch (IOException e) {
+ LOG.warn("Failed to write legacy OIV image: ", e);
+ }
+ }
return loadImage;
}
-
-
+
/**
* @param opts The parameters passed to this program.
* @exception Exception if the filesystem does not exist.
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java Thu May 15 13:03:22 2014
@@ -34,8 +34,8 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.NameNodeProxies;
+import org.apache.hadoop.hdfs.server.namenode.ha.AbstractNNFailoverProxyProvider;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
-import org.apache.hadoop.io.retry.FailoverProxyProvider;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.security.UserGroupInformation;
@@ -46,8 +46,8 @@ import com.google.common.base.Preconditi
* to connect to during fail-over. The first configured address is tried first,
* and on a fail-over event the other address is tried.
*/
-public class ConfiguredFailoverProxyProvider<T> implements
- FailoverProxyProvider<T> {
+public class ConfiguredFailoverProxyProvider<T> extends
+ AbstractNNFailoverProxyProvider<T> {
private static final Log LOG =
LogFactory.getLog(ConfiguredFailoverProxyProvider.class);
@@ -165,4 +165,12 @@ public class ConfiguredFailoverProxyProv
}
}
}
+
+ /**
+ * Logical URI is required for this failover proxy provider.
+ */
+ @Override
+ public boolean useLogicalURI() {
+ return true;
+ }
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java Thu May 15 13:03:22 2014
@@ -183,6 +183,12 @@ public class StandbyCheckpointer {
txid = img.getStorage().getMostRecentCheckpointTxId();
assert txid == thisCheckpointTxId : "expected to save checkpoint at txid=" +
thisCheckpointTxId + " but instead saved at txid=" + txid;
+
+ // Save the legacy OIV image, if the output dir is defined.
+ String outputDir = checkpointConf.getLegacyOivImageDir();
+ if (outputDir != null && !outputDir.isEmpty()) {
+ img.saveLegacyOIVImage(namesystem, outputDir, canceler);
+ }
} finally {
namesystem.longReadUnlock();
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java Thu May 15 13:03:22 2014
@@ -17,13 +17,17 @@
*/
package org.apache.hadoop.hdfs.server.namenode.snapshot;
-import com.google.common.base.Preconditions;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.List;
+
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeAttributes;
import org.apache.hadoop.hdfs.server.namenode.Quota;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
-import java.util.List;
+import com.google.common.base.Preconditions;
/**
* The difference of an inode between in two snapshots.
@@ -128,4 +132,11 @@ abstract class AbstractINodeDiff<N exten
return getClass().getSimpleName() + ": " + this.getSnapshotId() + " (post="
+ (posteriorDiff == null? null: posteriorDiff.getSnapshotId()) + ")";
}
+
+ void writeSnapshot(DataOutput out) throws IOException {
+ out.writeInt(snapshotId);
+ }
+
+ abstract void write(DataOutput out, ReferenceMap referenceMap
+ ) throws IOException;
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java Thu May 15 13:03:22 2014
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.namenode.snapshot;
+import java.io.DataOutput;
+import java.io.IOException;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
@@ -32,6 +34,7 @@ import org.apache.hadoop.hdfs.protocol.S
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.server.namenode.Content;
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
+import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
@@ -39,6 +42,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
import org.apache.hadoop.hdfs.server.namenode.Quota;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
import org.apache.hadoop.hdfs.util.Diff;
import org.apache.hadoop.hdfs.util.Diff.Container;
import org.apache.hadoop.hdfs.util.Diff.ListType;
@@ -120,6 +124,35 @@ public class DirectoryWithSnapshotFeatur
return counts;
}
+ /** Serialize {@link #created} */
+ private void writeCreated(DataOutput out) throws IOException {
+ final List<INode> created = getList(ListType.CREATED);
+ out.writeInt(created.size());
+ for (INode node : created) {
+ // For INode in created list, we only need to record its local name
+ byte[] name = node.getLocalNameBytes();
+ out.writeShort(name.length);
+ out.write(name);
+ }
+ }
+
+ /** Serialize {@link #deleted} */
+ private void writeDeleted(DataOutput out,
+ ReferenceMap referenceMap) throws IOException {
+ final List<INode> deleted = getList(ListType.DELETED);
+ out.writeInt(deleted.size());
+ for (INode node : deleted) {
+ FSImageSerialization.saveINode2Image(node, out, true, referenceMap);
+ }
+ }
+
+ /** Serialize to out */
+ private void write(DataOutput out, ReferenceMap referenceMap
+ ) throws IOException {
+ writeCreated(out);
+ writeDeleted(out, referenceMap);
+ }
+
/** Get the list of INodeDirectory contained in the deleted list */
private void getDirsInDeleted(List<INodeDirectory> dirList) {
for (INode node : getList(ListType.DELETED)) {
@@ -315,6 +348,25 @@ public class DirectoryWithSnapshotFeatur
}
@Override
+ void write(DataOutput out, ReferenceMap referenceMap) throws IOException {
+ writeSnapshot(out);
+ out.writeInt(childrenSize);
+
+ // Write snapshotINode
+ out.writeBoolean(isSnapshotRoot);
+ if (!isSnapshotRoot) {
+ if (snapshotINode != null) {
+ out.writeBoolean(true);
+ FSImageSerialization.writeINodeDirectoryAttributes(snapshotINode, out);
+ } else {
+ out.writeBoolean(false);
+ }
+ }
+ // Write diff. Node need to write poseriorDiff, since diffs is a list.
+ diff.write(out, referenceMap);
+ }
+
+ @Override
Quota.Counts destroyDiffAndCollectBlocks(INodeDirectory currentINode,
BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
// this diff has been deleted
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java Thu May 15 13:03:22 2014
@@ -17,13 +17,17 @@
*/
package org.apache.hadoop.hdfs.server.namenode.snapshot;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes;
import org.apache.hadoop.hdfs.server.namenode.Quota;
-
-import java.util.List;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
/**
* The difference of an {@link INodeFile} between two snapshots.
@@ -67,6 +71,20 @@ public class FileDiff extends
}
@Override
+ void write(DataOutput out, ReferenceMap referenceMap) throws IOException {
+ writeSnapshot(out);
+ out.writeLong(fileSize);
+
+ // write snapshotINode
+ if (snapshotINode != null) {
+ out.writeBoolean(true);
+ FSImageSerialization.writeINodeFileAttributes(snapshotINode, out);
+ } else {
+ out.writeBoolean(false);
+ }
+ }
+
+ @Override
Quota.Counts destroyDiffAndCollectBlocks(INodeFile currentINode,
BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
return currentINode.getFileWithSnapshotFeature()
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java Thu May 15 13:03:22 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode.snapshot;
import java.io.DataInput;
+import java.io.DataOutput;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
@@ -30,6 +31,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.AclFeature;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
+import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.XAttrFeature;
@@ -227,4 +229,11 @@ public class Snapshot implements Compara
public String toString() {
return getClass().getSimpleName() + "." + root.getLocalName() + "(id=" + id + ")";
}
+
+ /** Serialize the fields to out */
+ void write(DataOutput out) throws IOException {
+ out.writeInt(id);
+ // write root
+ FSImageSerialization.writeINodeDirectory(root, out);
+ }
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java Thu May 15 13:03:22 2014
@@ -29,21 +29,75 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.server.namenode.INode;
+import org.apache.hadoop.hdfs.server.namenode.INodeAttributes;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes;
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList;
import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff;
import org.apache.hadoop.hdfs.util.Diff.ListType;
-import org.apache.hadoop.hdfs.server.namenode.FSImageFormat.Loader;
+import org.apache.hadoop.hdfs.util.ReadOnlyList;
/**
* A helper class defining static methods for reading/writing snapshot related
* information from/to FSImage.
*/
public class SnapshotFSImageFormat {
+ /**
+ * Save snapshots and snapshot quota for a snapshottable directory.
+ * @param current The directory that the snapshots belongs to.
+ * @param out The {@link DataOutput} to write.
+ * @throws IOException
+ */
+ public static void saveSnapshots(INodeDirectorySnapshottable current,
+ DataOutput out) throws IOException {
+ // list of snapshots in snapshotsByNames
+ ReadOnlyList<Snapshot> snapshots = current.getSnapshotsByNames();
+ out.writeInt(snapshots.size());
+ for (Snapshot s : snapshots) {
+ // write the snapshot id
+ out.writeInt(s.getId());
+ }
+ // snapshot quota
+ out.writeInt(current.getSnapshotQuota());
+ }
+
+ /**
+ * Save SnapshotDiff list for an INodeDirectoryWithSnapshot.
+ * @param sNode The directory that the SnapshotDiff list belongs to.
+ * @param out The {@link DataOutput} to write.
+ */
+ private static <N extends INode, A extends INodeAttributes, D extends AbstractINodeDiff<N, A, D>>
+ void saveINodeDiffs(final AbstractINodeDiffList<N, A, D> diffs,
+ final DataOutput out, ReferenceMap referenceMap) throws IOException {
+ // Record the diffs in reversed order, so that we can find the correct
+ // reference for INodes in the created list when loading the FSImage
+ if (diffs == null) {
+ out.writeInt(-1); // no diffs
+ } else {
+ final List<D> list = diffs.asList();
+ final int size = list.size();
+ out.writeInt(size);
+ for (int i = size - 1; i >= 0; i--) {
+ list.get(i).write(out, referenceMap);
+ }
+ }
+ }
+
+ public static void saveDirectoryDiffList(final INodeDirectory dir,
+ final DataOutput out, final ReferenceMap referenceMap
+ ) throws IOException {
+ saveINodeDiffs(dir.getDiffs(), out, referenceMap);
+ }
+
+ public static void saveFileDiffList(final INodeFile file,
+ final DataOutput out) throws IOException {
+ saveINodeDiffs(file.getDiffs(), out, null);
+ }
+
public static FileDiffList loadFileDiffList(DataInput in,
FSImageFormat.Loader loader) throws IOException {
final int size = in.readInt();
@@ -264,6 +318,23 @@ public class SnapshotFSImageFormat {
* Used to record whether the subtree of the reference node has been saved
*/
private final Map<Long, Long> dirMap = new HashMap<Long, Long>();
+
+ public void writeINodeReferenceWithCount(
+ INodeReference.WithCount withCount, DataOutput out,
+ boolean writeUnderConstruction) throws IOException {
+ final INode referred = withCount.getReferredINode();
+ final long id = withCount.getId();
+ final boolean firstReferred = !referenceMap.containsKey(id);
+ out.writeBoolean(firstReferred);
+
+ if (firstReferred) {
+ FSImageSerialization.saveINode2Image(referred, out,
+ writeUnderConstruction, this);
+ referenceMap.put(id, withCount);
+ } else {
+ out.writeLong(id);
+ }
+ }
public boolean toProcessSubtree(long id) {
if (dirMap.containsKey(id)) {
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java Thu May 15 13:03:22 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode.snapshot;
import java.io.DataInput;
+import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -287,6 +288,22 @@ public class SnapshotManager implements
return snapshottables.values().toArray(
new INodeDirectorySnapshottable[snapshottables.size()]);
}
+
+ /**
+ * Write {@link #snapshotCounter}, {@link #numSnapshots},
+ * and all snapshots to the DataOutput.
+ */
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(snapshotCounter);
+ out.writeInt(numSnapshots.get());
+
+ // write all snapshots.
+ for(INodeDirectorySnapshottable snapshottableDir : snapshottables.values()) {
+ for(Snapshot s : snapshottableDir.getSnapshotsByNames()) {
+ s.write(out);
+ }
+ }
+ }
/**
* Read values of {@link #snapshotCounter}, {@link #numSnapshots}, and
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java Thu May 15 13:03:22 2014
@@ -913,9 +913,10 @@ public class DFSAdmin extends FsShell {
Configuration dfsConf = dfs.getConf();
URI dfsUri = dfs.getUri();
- boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri);
- if (isHaEnabled) {
- // In the case of HA, run finalizeUpgrade for all NNs in this nameservice
+ boolean isHaAndLogicalUri = HAUtil.isLogicalUri(dfsConf, dfsUri);
+ if (isHaAndLogicalUri) {
+ // In the case of HA and logical URI, run finalizeUpgrade for all
+ // NNs in this nameservice.
String nsId = dfsUri.getHost();
List<ClientProtocol> namenodes =
HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf, nsId);
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java Thu May 15 13:03:22 2014
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.tools;
import java.io.PrintStream;
import java.util.Arrays;
+import java.util.Collection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -27,6 +28,7 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.ha.HAAdmin;
import org.apache.hadoop.ha.HAServiceTarget;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.util.ToolRunner;
@@ -117,7 +119,15 @@ public class DFSHAAdmin extends HAAdmin
return super.runCmd(argv);
}
-
+
+ /**
+ * returns the list of all namenode ids for the given configuration
+ */
+ @Override
+ protected Collection<String> getTargetIds(String namenodeToActivate) {
+ return DFSUtil.getNameNodeIds(getConf(), (nameserviceId != null)? nameserviceId : DFSUtil.getNamenodeNameServiceId(getConf()));
+ }
+
public static void main(String[] argv) throws Exception {
int res = ToolRunner.run(new DFSHAAdmin(), argv);
System.exit(res);
Modified: hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java Thu May 15 13:03:22 2014
@@ -176,8 +176,9 @@ public class JsonUtil {
}
/** Convert a string to a FsPermission object. */
- private static FsPermission toFsPermission(final String s) {
- return new FsPermission(Short.parseShort(s, 8));
+ private static FsPermission toFsPermission(final String s, Boolean aclBit) {
+ FsPermission perm = new FsPermission(Short.parseShort(s, 8));
+ return (aclBit != null && aclBit) ? new FsAclPermission(perm) : perm;
}
static enum PathType {
@@ -204,7 +205,11 @@ public class JsonUtil {
m.put("length", status.getLen());
m.put("owner", status.getOwner());
m.put("group", status.getGroup());
- m.put("permission", toString(status.getPermission()));
+ FsPermission perm = status.getPermission();
+ m.put("permission", toString(perm));
+ if (perm.getAclBit()) {
+ m.put("aclBit", true);
+ }
m.put("accessTime", status.getAccessTime());
m.put("modificationTime", status.getModificationTime());
m.put("blockSize", status.getBlockSize());
@@ -230,7 +235,8 @@ public class JsonUtil {
final long len = (Long) m.get("length");
final String owner = (String) m.get("owner");
final String group = (String) m.get("group");
- final FsPermission permission = toFsPermission((String) m.get("permission"));
+ final FsPermission permission = toFsPermission((String) m.get("permission"),
+ (Boolean)m.get("aclBit"));
final long aTime = (Long) m.get("accessTime");
final long mTime = (Long) m.get("modificationTime");
final long blockSize = (Long) m.get("blockSize");