You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2012/08/27 03:07:05 UTC
svn commit: r1377548 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/common/
src/main/java/org/apache/hadoop/hdfs/server/datanode/
src/test/java/org/apache/hadoop/hdfs/ src/test/reso...
Author: eli
Date: Mon Aug 27 01:07:04 2012
New Revision: 1377548
URL: http://svn.apache.org/viewvc?rev=1377548&view=rev
Log:
HDFS-3731. Release upgrade must handle blocks being written from 1.0. Contributed by Colin Patrick McCabe
Added:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop1-bbw.tgz (with props)
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1377548&r1=1377547&r2=1377548&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Aug 27 01:07:04 2012
@@ -510,6 +510,9 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3683. Edit log replay progress indicator shows >100% complete. (Plamen
Jeliazkov via atm)
+ HDFS-3731. Release upgrade must handle blocks being written from 1.0.
+ (Colin Patrick McCabe via eli)
+
BREAKDOWN OF HDFS-3042 SUBTASKS
HDFS-2185. HDFS portion of ZK-based FailoverController (todd)
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1377548&r1=1377547&r2=1377548&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java Mon Aug 27 01:07:04 2012
@@ -85,6 +85,12 @@ public abstract class Storage extends St
public static final String STORAGE_TMP_LAST_CKPT = "lastcheckpoint.tmp";
public static final String STORAGE_PREVIOUS_CKPT = "previous.checkpoint";
+ /**
+ * The blocksBeingWritten directory which was used in some 1.x and earlier
+ * releases.
+ */
+ public static final String STORAGE_1_BBW = "blocksBeingWritten";
+
public enum StorageState {
NON_EXISTENT,
NOT_FORMATTED,
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java?rev=1377548&r1=1377547&r2=1377548&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java Mon Aug 27 01:07:04 2012
@@ -451,6 +451,8 @@ public class DataStorage extends Storage
File curDir = sd.getCurrentDir();
File prevDir = sd.getPreviousDir();
+ File bbwDir = new File(sd.getRoot(), Storage.STORAGE_1_BBW);
+
assert curDir.exists() : "Data node current directory must exist.";
// Cleanup directory "detach"
cleanupDetachDir(new File(curDir, STORAGE_DIR_DETACHED));
@@ -471,7 +473,7 @@ public class DataStorage extends Storage
BlockPoolSliceStorage bpStorage = new BlockPoolSliceStorage(nsInfo.getNamespaceID(),
nsInfo.getBlockPoolID(), nsInfo.getCTime(), nsInfo.getClusterID());
bpStorage.format(curDir, nsInfo);
- linkAllBlocks(tmpDir, new File(curBpDir, STORAGE_DIR_CURRENT));
+ linkAllBlocks(tmpDir, bbwDir, new File(curBpDir, STORAGE_DIR_CURRENT));
// 4. Write version file under <SD>/current
layoutVersion = HdfsConstants.LAYOUT_VERSION;
@@ -578,15 +580,21 @@ public class DataStorage extends Storage
+ "; cur CTime = " + this.getCTime());
assert sd.getCurrentDir().exists() : "Current directory must exist.";
final File tmpDir = sd.getFinalizedTmp();//finalized.tmp directory
+ final File bbwDir = new File(sd.getRoot(), Storage.STORAGE_1_BBW);
// 1. rename previous to finalized.tmp
rename(prevDir, tmpDir);
// 2. delete finalized.tmp dir in a separate thread
+ // Also delete the blocksBeingWritten from HDFS 1.x and earlier, if
+ // it exists.
new Daemon(new Runnable() {
@Override
public void run() {
try {
deleteDir(tmpDir);
+ if (bbwDir.exists()) {
+ deleteDir(bbwDir);
+ }
} catch(IOException ex) {
LOG.error("Finalize upgrade for " + dataDirPath + " failed.", ex);
}
@@ -620,11 +628,16 @@ public class DataStorage extends Storage
/**
* Hardlink all finalized and RBW blocks in fromDir to toDir
- * @param fromDir directory where the snapshot is stored
- * @param toDir the current data directory
- * @throws IOException if error occurs during hardlink
+ *
+ * @param fromDir The directory where the 'from' snapshot is stored
+ * @param fromBbwDir In HDFS 1.x, the directory where blocks
+ * that are under construction are stored.
+ * @param toDir The current data directory
+ *
+ * @throws IOException If error occurs during hardlink
*/
- private void linkAllBlocks(File fromDir, File toDir) throws IOException {
+ private void linkAllBlocks(File fromDir, File fromBbwDir, File toDir)
+ throws IOException {
HardLink hardLink = new HardLink();
// do the link
int diskLayoutVersion = this.getLayoutVersion();
@@ -632,13 +645,23 @@ public class DataStorage extends Storage
// hardlink finalized blocks in tmpDir/finalized
linkBlocks(new File(fromDir, STORAGE_DIR_FINALIZED),
new File(toDir, STORAGE_DIR_FINALIZED), diskLayoutVersion, hardLink);
- // hardlink rbw blocks in tmpDir/finalized
+ // hardlink rbw blocks in tmpDir/rbw
linkBlocks(new File(fromDir, STORAGE_DIR_RBW),
new File(toDir, STORAGE_DIR_RBW), diskLayoutVersion, hardLink);
} else { // pre-RBW version
// hardlink finalized blocks in tmpDir
linkBlocks(fromDir, new File(toDir, STORAGE_DIR_FINALIZED),
diskLayoutVersion, hardLink);
+ if (fromBbwDir.exists()) {
+ /*
+ * We need to put the 'blocksBeingWritten' from HDFS 1.x into the rbw
+ * directory. It's a little messy, because the blocksBeingWriten was
+ * NOT underneath the 'current' directory in those releases. See
+ * HDFS-3731 for details.
+ */
+ linkBlocks(fromBbwDir,
+ new File(toDir, STORAGE_DIR_RBW), diskLayoutVersion, hardLink);
+ }
}
LOG.info( hardLink.linkStats.report() );
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java?rev=1377548&r1=1377547&r2=1377548&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java Mon Aug 27 01:07:04 2012
@@ -39,7 +39,9 @@ import org.apache.hadoop.fs.FSInputStrea
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
import org.apache.hadoop.util.StringUtils;
@@ -49,8 +51,9 @@ import org.junit.Test;
* This tests data transfer protocol handling in the Datanode. It sends
* various forms of wrong data and verifies that Datanode handles it well.
*
- * This test uses the following two file from src/test/.../dfs directory :
- * 1) hadoop-version-dfs-dir.tgz : contains DFS directories.
+ * This test uses the following items from src/test/.../dfs directory :
+ * 1) hadoop-22-dfs-dir.tgz and other tarred pre-upgrade NN / DN
+ * directory images
* 2) hadoop-dfs-dir.txt : checksums that are compared in this test.
* Please read hadoop-dfs-dir.txt for more information.
*/
@@ -62,14 +65,23 @@ public class TestDFSUpgradeFromImage {
new File(MiniDFSCluster.getBaseDirectory());
private static final String HADOOP_DFS_DIR_TXT = "hadoop-dfs-dir.txt";
private static final String HADOOP22_IMAGE = "hadoop-22-dfs-dir.tgz";
-
- public int numDataNodes = 4;
-
+ private static final String HADOOP1_BBW_IMAGE = "hadoop1-bbw.tgz";
+
private static class ReferenceFileInfo {
String path;
long checksum;
}
+ private static final Configuration upgradeConf;
+
+ static {
+ upgradeConf = new HdfsConfiguration();
+ upgradeConf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1); // block scanning off
+ if (System.getProperty("test.build.data") == null) { // to allow test to be run outside of Maven
+ System.setProperty("test.build.data", "build/test/data");
+ }
+ }
+
LinkedList<ReferenceFileInfo> refList = new LinkedList<ReferenceFileInfo>();
Iterator<ReferenceFileInfo> refIter;
@@ -137,11 +149,33 @@ public class TestDFSUpgradeFromImage {
}
}
- CRC32 overallChecksum = new CRC32();
+ /**
+ * Try to open a file for reading several times.
+ *
+ * If we fail because lease recovery hasn't completed, retry the open.
+ */
+ private static FSInputStream dfsOpenFileWithRetries(DistributedFileSystem dfs,
+ String pathName) throws IOException {
+ IOException exc = null;
+ for (int tries = 0; tries < 10; tries++) {
+ try {
+ return dfs.dfs.open(pathName);
+ } catch (IOException e) {
+ exc = e;
+ }
+ if (!exc.getMessage().contains("Cannot obtain " +
+ "block length for LocatedBlock")) {
+ throw exc;
+ }
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ignored) {}
+ }
+ throw exc;
+ }
- private void verifyDir(DistributedFileSystem dfs, Path dir)
- throws IOException {
-
+ private void verifyDir(DistributedFileSystem dfs, Path dir,
+ CRC32 overallChecksum) throws IOException {
FileStatus[] fileArr = dfs.listStatus(dir);
TreeMap<Path, Boolean> fileMap = new TreeMap<Path, Boolean>();
@@ -157,11 +191,11 @@ public class TestDFSUpgradeFromImage {
overallChecksum.update(pathName.getBytes());
if ( isDir ) {
- verifyDir(dfs, path);
+ verifyDir(dfs, path, overallChecksum);
} else {
// this is not a directory. Checksum the file data.
CRC32 fileCRC = new CRC32();
- FSInputStream in = dfs.dfs.open(pathName);
+ FSInputStream in = dfsOpenFileWithRetries(dfs, pathName);
byte[] buf = new byte[4096];
int nRead = 0;
while ( (nRead = in.read(buf, 0, buf.length)) > 0 ) {
@@ -175,7 +209,8 @@ public class TestDFSUpgradeFromImage {
private void verifyFileSystem(DistributedFileSystem dfs) throws IOException {
- verifyDir(dfs, new Path("/"));
+ CRC32 overallChecksum = new CRC32();
+ verifyDir(dfs, new Path("/"), overallChecksum);
verifyChecksum("overallCRC", overallChecksum.getValue());
@@ -237,7 +272,8 @@ public class TestDFSUpgradeFromImage {
@Test
public void testUpgradeFromRel22Image() throws IOException {
unpackStorage(HADOOP22_IMAGE);
- upgradeAndVerify();
+ upgradeAndVerify(new MiniDFSCluster.Builder(upgradeConf).
+ numDataNodes(4));
}
/**
@@ -259,7 +295,8 @@ public class TestDFSUpgradeFromImage {
// Upgrade should now fail
try {
- upgradeAndVerify();
+ upgradeAndVerify(new MiniDFSCluster.Builder(upgradeConf).
+ numDataNodes(4));
fail("Upgrade did not fail with bad MD5");
} catch (IOException ioe) {
String msg = StringUtils.stringifyException(ioe);
@@ -268,21 +305,34 @@ public class TestDFSUpgradeFromImage {
}
}
}
-
- private void upgradeAndVerify() throws IOException {
+
+ static void recoverAllLeases(DFSClient dfs,
+ Path path) throws IOException {
+ String pathStr = path.toString();
+ HdfsFileStatus status = dfs.getFileInfo(pathStr);
+ if (!status.isDir()) {
+ dfs.recoverLease(pathStr);
+ return;
+ }
+ byte prev[] = HdfsFileStatus.EMPTY_NAME;
+ DirectoryListing dirList;
+ do {
+ dirList = dfs.listPaths(pathStr, prev);
+ HdfsFileStatus files[] = dirList.getPartialListing();
+ for (HdfsFileStatus f : files) {
+ recoverAllLeases(dfs, f.getFullPath(path));
+ }
+ prev = dirList.getLastName();
+ } while (dirList.hasMore());
+ }
+
+ private void upgradeAndVerify(MiniDFSCluster.Builder bld)
+ throws IOException {
MiniDFSCluster cluster = null;
try {
- Configuration conf = new HdfsConfiguration();
- if (System.getProperty("test.build.data") == null) { // to allow test to be run outside of Ant
- System.setProperty("test.build.data", "build/test/data");
- }
- conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1); // block scanning off
- cluster = new MiniDFSCluster.Builder(conf)
- .numDataNodes(numDataNodes)
- .format(false)
- .startupOption(StartupOption.UPGRADE)
- .clusterId("testClusterId")
- .build();
+ bld.format(false).startupOption(StartupOption.UPGRADE)
+ .clusterId("testClusterId");
+ cluster = bld.build();
cluster.waitActive();
DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem();
DFSClient dfsClient = dfs.dfs;
@@ -293,12 +343,27 @@ public class TestDFSUpgradeFromImage {
Thread.sleep(1000);
} catch (InterruptedException ignored) {}
}
-
+ recoverAllLeases(dfsClient, new Path("/"));
verifyFileSystem(dfs);
} finally {
if (cluster != null) { cluster.shutdown(); }
}
}
-
+ /**
+ * Test upgrade from a 1.x image with some blocksBeingWritten
+ */
+ @Test
+ public void testUpgradeFromRel1BBWImage() throws IOException {
+ unpackStorage(HADOOP1_BBW_IMAGE);
+ Configuration conf = new Configuration(upgradeConf);
+ conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
+ System.getProperty("test.build.data") + File.separator +
+ "dfs" + File.separator +
+ "data" + File.separator +
+ "data1");
+ upgradeAndVerify(new MiniDFSCluster.Builder(conf).
+ numDataNodes(1).enableManagedDfsDirsRedundancy(false).
+ manageDataDfsDirs(false));
+ }
}
Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop1-bbw.tgz
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop1-bbw.tgz?rev=1377548&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop1-bbw.tgz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream