You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ha...@apache.org on 2011/04/15 00:16:33 UTC
svn commit: r1092534 - in /hadoop/hdfs/trunk: ./
src/java/org/apache/hadoop/hdfs/protocol/
src/java/org/apache/hadoop/hdfs/server/namenode/
src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/
src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/
Author: hairong
Date: Thu Apr 14 22:16:32 2011
New Revision: 1092534
URL: http://svn.apache.org/viewvc?rev=1092534&view=rev
Log:
HDFS-1070. Speedup namenode image loading and saving by storing only local file names. Contributed by Hairong Kuang.
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1092534&r1=1092533&r2=1092534&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Thu Apr 14 22:16:32 2011
@@ -32,6 +32,9 @@ Trunk (unreleased changes)
HDFS-1442. Api to get delegation token in Hdfs class. (jitendra)
+ HDFS-1070. Speedup namenode image loading and saving by storing only
+ local file names. (hairong)
+
IMPROVEMENTS
HDFS-1510. Added test-patch.properties required by test-patch.sh (nigel)
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java?rev=1092534&r1=1092533&r2=1092534&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java Thu Apr 14 22:16:32 2011
@@ -88,7 +88,7 @@ public interface FSConstants {
// Version is reflected in the data storage file.
// Versions are negative.
// Decrement LAYOUT_VERSION to define a new version.
- public static final int LAYOUT_VERSION = -28;
+ public static final int LAYOUT_VERSION = -30;
// Current version:
- // -28: Support fsedits checksum
+ // -30: store only last component of a path in image
}
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1092534&r1=1092533&r2=1092534&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java Thu Apr 14 22:16:32 2011
@@ -30,6 +30,7 @@ import java.security.DigestInputStream;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
import java.util.Arrays;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -179,7 +180,11 @@ class FSImageFormat {
// load all inodes
LOG.info("Number of files = " + numFiles);
- loadFullNameINodes(numFiles, in);
+ if (imgVersion <= -30) {
+ loadLocalNameINodes(numFiles, in);
+ } else {
+ loadFullNameINodes(numFiles, in);
+ }
// load datanode info
this.loadDatanodes(in);
@@ -215,6 +220,64 @@ class FSImageFormat {
fsDir.rootDir.setPermissionStatus(root.getPermissionStatus());
}
+ /**
+ * load fsimage files assuming only local names are stored
+ *
+ * @param numFiles number of files expected to be read
+ * @param in image input stream
+ * @throws IOException
+ */
+ private void loadLocalNameINodes(long numFiles, DataInputStream in)
+ throws IOException {
+ assert imgVersion <= -30; // -30: store only local name in image
+ assert numFiles > 0;
+
+ // load root
+ if( in.readShort() != 0) {
+ throw new IOException("First node is not root");
+ }
+ INode root = loadINode(in);
+ // update the root's attributes
+ updateRootAttr(root);
+ numFiles--;
+
+ // load rest of the nodes directory by directory
+ while (numFiles > 0) {
+ numFiles -= loadDirectory(in);
+ }
+ if (numFiles != 0) {
+ throw new IOException("Read unexpect number of files: " + -numFiles);
+ }
+ }
+
+ /**
+ * Load all children of a directory
+ *
+ * @param in
+ * @return number of child inodes read
+ * @throws IOException
+ */
+ private int loadDirectory(DataInputStream in) throws IOException {
+ String parentPath = FSImageSerialization.readString(in);
+ FSDirectory fsDir = namesystem.dir;
+ INode parent = fsDir.rootDir.getNode(parentPath, true);
+ if (parent == null || !parent.isDirectory()) {
+ throw new IOException("Path " + parentPath + "is not a directory.");
+ }
+
+ int numChildren = in.readInt();
+ for(int i=0; i<numChildren; i++) {
+ // load single inode
+ byte[] localName = new byte[in.readShort()];
+ in.readFully(localName); // read local name
+ INode newNode = loadINode(in); // read rest of inode
+
+ // add to parent
+ namesystem.dir.addToParent(localName, (INodeDirectory)parent, newNode, false);
+ }
+ return numChildren;
+ }
+
/**
* load fsimage files assuming full path names are stored
*
@@ -485,9 +548,10 @@ class FSImageFormat {
byte[] byteStore = new byte[4*FSConstants.MAX_PATH_LENGTH];
ByteBuffer strbuf = ByteBuffer.wrap(byteStore);
// save the root
- FSImageSerialization.saveINode2Image(strbuf, fsDir.rootDir, out);
+ FSImageSerialization.saveINode2Image(fsDir.rootDir, out);
// save the rest of the nodes
- saveImage(strbuf, 0, fsDir.rootDir, out);
+ saveImage(strbuf, fsDir.rootDir, out);
+ // save files under construction
sourceNamesystem.saveFilesUnderConstruction(out);
sourceNamesystem.saveSecretManagerState(out);
strbuf = null;
@@ -511,28 +575,33 @@ class FSImageFormat {
* This is a recursive procedure, which first saves all children of
* a current directory and then moves inside the sub-directories.
*/
- private static void saveImage(ByteBuffer parentPrefix,
- int prefixLength,
+ private static void saveImage(ByteBuffer currentDirName,
INodeDirectory current,
DataOutputStream out) throws IOException {
- int newPrefixLength = prefixLength;
- if (current.getChildrenRaw() == null)
+ List<INode> children = current.getChildrenRaw();
+ if (children == null || children.isEmpty())
return;
- for(INode child : current.getChildren()) {
+ // print prefix (parent directory name)
+ int prefixLen = currentDirName.position();
+ if (prefixLen == 0) { // root
+ out.writeShort(PATH_SEPARATOR.length);
+ out.write(PATH_SEPARATOR);
+ } else { // non-root directories
+ out.writeShort(prefixLen);
+ out.write(currentDirName.array(), 0, prefixLen);
+ }
+ out.writeInt(children.size());
+ for(INode child : children) {
// print all children first
- parentPrefix.position(prefixLength);
- parentPrefix.put(PATH_SEPARATOR).put(child.getLocalNameBytes());
- FSImageSerialization.saveINode2Image(parentPrefix, child, out);
+ FSImageSerialization.saveINode2Image(child, out);
}
- for(INode child : current.getChildren()) {
+ for(INode child : children) {
if(!child.isDirectory())
continue;
- parentPrefix.position(prefixLength);
- parentPrefix.put(PATH_SEPARATOR).put(child.getLocalNameBytes());
- newPrefixLength = parentPrefix.position();
- saveImage(parentPrefix, newPrefixLength, (INodeDirectory)child, out);
+ currentDirName.put(PATH_SEPARATOR).put(child.getLocalNameBytes());
+ saveImage(currentDirName, (INodeDirectory)child, out);
+ currentDirName.position(prefixLen);
}
- parentPrefix.position(prefixLength);
}
}
}
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java?rev=1092534&r1=1092533&r2=1092534&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java Thu Apr 14 22:16:32 2011
@@ -145,12 +145,11 @@ public class FSImageSerialization {
/*
* Save one inode's attributes to the image.
*/
- static void saveINode2Image(ByteBuffer name,
- INode node,
+ static void saveINode2Image(INode node,
DataOutputStream out) throws IOException {
- int nameLen = name.position();
- out.writeShort(nameLen);
- out.write(name.array(), name.arrayOffset(), nameLen);
+ byte[] name = node.getLocalNameBytes();
+ out.writeShort(name.length);
+ out.write(name);
FsPermission filePerm = TL_DATA.get().FILE_PERM;
if (node.isDirectory()) {
out.writeShort(0); // replication
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java?rev=1092534&r1=1092533&r2=1092534&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java Thu Apr 14 22:16:32 2011
@@ -38,7 +38,7 @@ import static org.apache.hadoop.hdfs.too
class EditsLoaderCurrent implements EditsLoader {
private static int [] supportedVersions = {
- -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28 };
+ -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -30 };
private EditsVisitor v;
private int editsVersion = 0;
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java?rev=1092534&r1=1092533&r2=1092534&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java Thu Apr 14 22:16:32 2011
@@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.tools.offlineImageViewer.ImageVisitor.ImageElement;
import org.apache.hadoop.io.Text;
@@ -121,7 +120,7 @@ class ImageLoaderCurrent implements Imag
protected final DateFormat dateFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm");
private static int [] versions =
- {-16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28};
+ {-16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -30};
private int imageVersion = 0;
/* (non-Javadoc)
@@ -334,34 +333,105 @@ class ImageLoaderCurrent implements Imag
long numInodes, boolean skipBlocks) throws IOException {
v.visitEnclosingElement(ImageElement.INODES,
ImageElement.NUM_INODES, numInodes);
+
+ if (imageVersion <= -30) { // local file name
+ processLocalNameINodes(in, v, numInodes, skipBlocks);
+ } else { // full path name
+ processFullNameINodes(in, v, numInodes, skipBlocks);
+ }
- for(long i = 0; i < numInodes; i++) {
- v.visitEnclosingElement(ImageElement.INODE);
- v.visit(ImageElement.INODE_PATH, FSImageSerialization.readString(in));
- v.visit(ImageElement.REPLICATION, in.readShort());
- v.visit(ImageElement.MODIFICATION_TIME, formatDate(in.readLong()));
- if(imageVersion <= -17) // added in version -17
- v.visit(ImageElement.ACCESS_TIME, formatDate(in.readLong()));
- v.visit(ImageElement.BLOCK_SIZE, in.readLong());
- int numBlocks = in.readInt();
-
- processBlocks(in, v, numBlocks, skipBlocks);
-
- // File or directory
- if (numBlocks > 0 || numBlocks == -1) {
- v.visit(ImageElement.NS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
- if(imageVersion <= -18) // added in version -18
- v.visit(ImageElement.DS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
- }
- if (imageVersion <= -23 && numBlocks == -2) {
- v.visit(ImageElement.SYMLINK, Text.readString(in));
+
+ v.leaveEnclosingElement(); // INodes
+ }
+
+ /**
+ * Process image with full path name
+ *
+ * @param in image stream
+ * @param v visitor
+ * @param numInodes number of indoes to read
+ * @param skipBlocks skip blocks or not
+ * @throws IOException if there is any error occurs
+ */
+ private void processLocalNameINodes(DataInputStream in, ImageVisitor v,
+ long numInodes, boolean skipBlocks) throws IOException {
+ // process root
+ processINode(in, v, skipBlocks, "");
+ numInodes--;
+ while (numInodes > 0) {
+ numInodes -= processDirectory(in, v, skipBlocks);
+ }
+ }
+
+ private int processDirectory(DataInputStream in, ImageVisitor v,
+ boolean skipBlocks) throws IOException {
+ String parentName = FSImageSerialization.readString(in);
+ int numChildren = in.readInt();
+ for (int i=0; i<numChildren; i++) {
+ processINode(in, v, skipBlocks, parentName);
+ }
+ return numChildren;
+ }
+
+ /**
+ * Process image with full path name
+ *
+ * @param in image stream
+ * @param v visitor
+ * @param numInodes number of indoes to read
+ * @param skipBlocks skip blocks or not
+ * @throws IOException if there is any error occurs
+ */
+ private void processFullNameINodes(DataInputStream in, ImageVisitor v,
+ long numInodes, boolean skipBlocks) throws IOException {
+ for(long i = 0; i < numInodes; i++) {
+ processINode(in, v, skipBlocks, null);
+ }
+ }
+
+ /**
+ * Process an INode
+ *
+ * @param in image stream
+ * @param v visitor
+ * @param skipBlocks skip blocks or not
+ * @param parentName the name of its parent node
+ * @return the number of Children
+ * @throws IOException
+ */
+ private void processINode(DataInputStream in, ImageVisitor v,
+ boolean skipBlocks, String parentName) throws IOException {
+ v.visitEnclosingElement(ImageElement.INODE);
+ String pathName = FSImageSerialization.readString(in);
+ if (parentName != null) { // local name
+ pathName = "/" + pathName;
+ if (!"/".equals(parentName)) { // children of non-root directory
+ pathName = parentName + pathName;
}
+ }
- processPermission(in, v);
- v.leaveEnclosingElement(); // INode
+ v.visit(ImageElement.INODE_PATH, pathName);
+ v.visit(ImageElement.REPLICATION, in.readShort());
+ v.visit(ImageElement.MODIFICATION_TIME, formatDate(in.readLong()));
+ if(imageVersion <= -17) // added in version -17
+ v.visit(ImageElement.ACCESS_TIME, formatDate(in.readLong()));
+ v.visit(ImageElement.BLOCK_SIZE, in.readLong());
+ int numBlocks = in.readInt();
+
+ processBlocks(in, v, numBlocks, skipBlocks);
+
+ // File or directory
+ if (numBlocks > 0 || numBlocks == -1) {
+ v.visit(ImageElement.NS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
+ if(imageVersion <= -18) // added in version -18
+ v.visit(ImageElement.DS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
}
-
- v.leaveEnclosingElement(); // INodes
+ if (imageVersion <= -23 && numBlocks == -2) {
+ v.visit(ImageElement.SYMLINK, Text.readString(in));
+ }
+
+ processPermission(in, v);
+ v.leaveEnclosingElement(); // INode
}
/**