You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by jg...@apache.org on 2011/03/18 20:14:16 UTC
svn commit: r1083021 - in /hadoop/hdfs/trunk: CHANGES.txt
src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend.java
Author: jghoman
Date: Fri Mar 18 19:14:16 2011
New Revision: 1083021
URL: http://svn.apache.org/viewvc?rev=1083021&view=rev
Log:
HDFS-1445. Batch the calls in DataStorage to FileUtil.createHardLink(). Contributed by Matt Foley.
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1083021&r1=1083020&r2=1083021&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Fri Mar 18 19:14:16 2011
@@ -73,6 +73,9 @@ Trunk (unreleased changes)
HDFS-780. Revive TestFuseDFS. (eli)
+ HDFS-1445. Batch the calls in DataStorage to FileUtil.createHardLink().
+ (Matt Foley via jghoman)
+
OPTIMIZATIONS
HDFS-1458. Improve checkpoint performance by avoiding unnecessary image
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java?rev=1083021&r1=1083020&r2=1083021&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java Fri Mar 18 19:14:16 2011
@@ -32,7 +32,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.FileUtil.HardLink;
+import org.apache.hadoop.fs.HardLink;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
@@ -402,23 +402,26 @@ public class DataStorage extends Storage
* @throws IOException if error occurs during hardlink
*/
private void linkAllBlocks(File fromDir, File toDir) throws IOException {
+ HardLink hardLink = new HardLink();
// do the link
int diskLayoutVersion = this.getLayoutVersion();
if (diskLayoutVersion < PRE_RBW_LAYOUT_VERSION) { // RBW version
// hardlink finalized blocks in tmpDir/finalized
linkBlocks(new File(fromDir, STORAGE_DIR_FINALIZED),
- new File(toDir, STORAGE_DIR_FINALIZED), diskLayoutVersion);
+ new File(toDir, STORAGE_DIR_FINALIZED), diskLayoutVersion, hardLink);
// hardlink rbw blocks in tmpDir/finalized
linkBlocks(new File(fromDir, STORAGE_DIR_RBW),
- new File(toDir, STORAGE_DIR_RBW), diskLayoutVersion);
+ new File(toDir, STORAGE_DIR_RBW), diskLayoutVersion, hardLink);
} else { // pre-RBW version
// hardlink finalized blocks in tmpDir
- linkBlocks(fromDir,
- new File(toDir, STORAGE_DIR_FINALIZED), diskLayoutVersion);
- }
+ linkBlocks(fromDir, new File(toDir, STORAGE_DIR_FINALIZED),
+ diskLayoutVersion, hardLink);
+ }
+ LOG.info( hardLink.linkStats.report() );
}
- static void linkBlocks(File from, File to, int oldLV) throws IOException {
+ static void linkBlocks(File from, File to, int oldLV, HardLink hl)
+ throws IOException {
if (!from.exists()) {
return;
}
@@ -429,6 +432,7 @@ public class DataStorage extends Storage
FileOutputStream out = new FileOutputStream(to);
try {
IOUtils.copyBytes(in, out, 16*1024);
+ hl.linkStats.countPhysicalFileCopies++;
} finally {
out.close();
}
@@ -444,23 +448,60 @@ public class DataStorage extends Storage
}
HardLink.createHardLink(from, to);
+ hl.linkStats.countSingleLinks++;
}
return;
}
// from is a directory
+ hl.linkStats.countDirs++;
+
if (!to.mkdirs())
throw new IOException("Cannot create directory " + to);
- String[] blockNames = from.list(new java.io.FilenameFilter() {
+
+ //If upgrading from old stuff, need to munge the filenames. That has to
+ //be done one file at a time, so hardlink them one at a time (slow).
+ if (oldLV >= PRE_GENERATIONSTAMP_LAYOUT_VERSION) {
+ String[] blockNames = from.list(new java.io.FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.startsWith(BLOCK_SUBDIR_PREFIX)
+ || name.startsWith(BLOCK_FILE_PREFIX)
+ || name.startsWith(COPY_FILE_PREFIX);
+ }
+ });
+ if (blockNames.length == 0) {
+ hl.linkStats.countEmptyDirs++;
+ }
+ else for(int i = 0; i < blockNames.length; i++)
+ linkBlocks(new File(from, blockNames[i]),
+ new File(to, blockNames[i]), oldLV, hl);
+ }
+ else {
+ //If upgrading from a relatively new version, we only need to create
+ //links with the same filename. This can be done in bulk (much faster).
+ String[] blockNames = from.list(new java.io.FilenameFilter() {
public boolean accept(File dir, String name) {
- return name.startsWith(BLOCK_SUBDIR_PREFIX)
- || name.startsWith(BLOCK_FILE_PREFIX)
- || name.startsWith(COPY_FILE_PREFIX);
+ return name.startsWith(BLOCK_FILE_PREFIX);
}
});
-
- for(int i = 0; i < blockNames.length; i++)
- linkBlocks(new File(from, blockNames[i]),
- new File(to, blockNames[i]), oldLV);
+ if (blockNames.length > 0) {
+ HardLink.createHardLinkMult(from, blockNames, to);
+ hl.linkStats.countMultLinks++;
+ hl.linkStats.countFilesMultLinks += blockNames.length;
+ } else {
+ hl.linkStats.countEmptyDirs++;
+ }
+
+ //now take care of the rest of the files and subdirectories
+ String[] otherNames = from.list(new java.io.FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.startsWith(BLOCK_SUBDIR_PREFIX)
+ || name.startsWith(COPY_FILE_PREFIX);
+ }
+ });
+ for(int i = 0; i < otherNames.length; i++)
+ linkBlocks(new File(from, otherNames[i]),
+ new File(to, otherNames[i]), oldLV, hl);
+ }
}
private void verifyDistributedUpgradeProgress(
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java?rev=1083021&r1=1083020&r2=1083021&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java Fri Mar 18 19:14:16 2011
@@ -24,7 +24,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.FileUtil.HardLink;
+import org.apache.hadoop.fs.HardLink;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.datanode.FSDataset.FSVolume;
import org.apache.hadoop.io.IOUtils;
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend.java?rev=1083021&r1=1083020&r2=1083021&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend.java Fri Mar 18 19:14:16 2011
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.BlockLocatio
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileUtil.HardLink;
+import org.apache.hadoop.fs.HardLink;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;