You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2011/03/02 21:02:12 UTC
svn commit: r1076358 - in /hadoop/hdfs/branches/HDFS-1052: ./
src/java/org/apache/hadoop/hdfs/server/common/
src/java/org/apache/hadoop/hdfs/server/datanode/
src/java/org/apache/hadoop/hdfs/server/namenode/
src/test/aop/org/apache/hadoop/fs/ src/test/a...
Author: suresh
Date: Wed Mar 2 20:02:12 2011
New Revision: 1076358
URL: http://svn.apache.org/viewvc?rev=1076358&view=rev
Log:
HDFS-1688. Fix failures in fault injection tests, TestDiskError, TestDatanodeRestart and TestDFSTartupVersions. Contributed by Suresh Srinivas.
Modified:
hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/fs/TestFiRename.java
hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java
hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol2.java
hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSStartupVersions.java
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestMulitipleNNDataBlockScanner.java
Modified: hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/CHANGES.txt?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/CHANGES.txt (original)
+++ hadoop/hdfs/branches/HDFS-1052/CHANGES.txt Wed Mar 2 20:02:12 2011
@@ -123,6 +123,9 @@ Trunk (unreleased changes)
HDFS-1678. Federation: Remove unnecessary #getBlockpool()
for NameNodeMXBean in FSNameSystem. (Tanping via Suresh)
+ HDFS-1688. Fix failures in fault injection tests, TestDiskError,
+ TestDatanodeRestart and TestDFSTartupVersions. (suresh)
+
IMPROVEMENTS
HDFS-1510. Added test-patch.properties required by test-patch.sh (nigel)
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java Wed Mar 2 20:02:12 2011
@@ -728,8 +728,9 @@ public abstract class Storage extends St
protected static void checkVersionUpgradable(int oldVersion)
throws IOException {
if (oldVersion > LAST_UPGRADABLE_LAYOUT_VERSION) {
- String msg = "*********** Upgrade is not supported from this older" +
- " version of storage to the current version." +
+ String msg = "*********** Upgrade is not supported from this " +
+ " older version " + oldVersion +
+ " of storage to the current version." +
" Please upgrade to " + LAST_UPGRADABLE_HADOOP_VERSION +
" or a later version and then upgrade to current" +
" version. Old layout version is " +
@@ -773,7 +774,10 @@ public abstract class Storage extends St
props.setProperty("layoutVersion", String.valueOf(layoutVersion));
props.setProperty("storageType", storageType.toString());
props.setProperty("namespaceID", String.valueOf(namespaceID));
- props.setProperty("clusterID", clusterID);
+ // Set clusterID in version LAST_PRE_FEDERATION_LAYOUT_VERSION or before
+ if (layoutVersion < LAST_PRE_FEDERATION_LAYOUT_VERSION) {
+ props.setProperty("clusterID", clusterID);
+ }
props.setProperty("cTime", String.valueOf(cTime));
}
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java Wed Mar 2 20:02:12 2011
@@ -19,11 +19,9 @@ package org.apache.hadoop.hdfs.server.co
import java.io.DataInput;
import java.io.DataOutput;
-import java.io.File;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java Wed Mar 2 20:02:12 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
@@ -64,7 +65,13 @@ public class BlockPoolSliceStorage exten
super(NodeType.DATA_NODE);
}
- BlockPoolSliceStorage(int namespaceID, String bpID, long cTime, String clusterId) {
+ public BlockPoolSliceStorage(StorageInfo storageInfo, String bpid) {
+ super(NodeType.DATA_NODE, storageInfo);
+ blockpoolID = bpid;
+ }
+
+ BlockPoolSliceStorage(int namespaceID, String bpID, long cTime,
+ String clusterId) {
super(NodeType.DATA_NODE);
this.namespaceID = namespaceID;
this.blockpoolID = bpID;
@@ -513,10 +520,10 @@ public class BlockPoolSliceStorage exten
/**
* Get a block pool storage root based on data node storage root
* @param bpID block pool ID
- * @param dnRoot data node storage root directory
+ * @param dnCurDir data node storage root directory
* @return root directory for block pool storage
*/
- static File getBpRoot(String bpID, File dnCurDir) {
+ public static File getBpRoot(String bpID, File dnCurDir) {
return new File(dnCurDir, bpID);
}
}
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Wed Mar 2 20:02:12 2011
@@ -319,7 +319,7 @@ public class DataNode extends Configured
volatile boolean shouldRun = true;
private BlockPoolManager blockPoolManager;
public DatanodeProtocol namenodeTODO_FED = null; //TODO:FEDERATION needs to be taken out.
- public FSDatasetInterface data = null;
+ public volatile FSDatasetInterface data = null;
private DatanodeID datanodeId = null;
private String clusterId = null;
@@ -570,7 +570,7 @@ public class DataNode extends Configured
}
}
- private void shutdownDirectoryScanner() {
+ private synchronized void shutdownDirectoryScanner() {
if (directoryScanner != null) {
directoryScanner.shutdown();
}
@@ -725,12 +725,10 @@ public class DataNode extends Configured
DatanodeProtocol dnp =
(DatanodeProtocol)RPC.waitForProxy(DatanodeProtocol.class,
DatanodeProtocol.versionID, nnAddr, conf);
- LOG.info("NN proxy created in BP="+blockPoolId + " for " + nnAddr);
setNameNode(dnp);
// handshake with NN
NamespaceInfo nsInfo = handshake();
- LOG.info("received namespace info nsInfo=" + nsInfo);
setNamespaceInfo(nsInfo);
setClusterId(nsInfo.clusterID);
@@ -1654,7 +1652,7 @@ public class DataNode extends Configured
}
LOG.warn("DataNode is shutting down.\n" + errMsgr);
- shutdown();
+ shouldRun = false;
}
/** Number of concurrent xceivers per node. */
@@ -1937,9 +1935,11 @@ public class DataNode extends Configured
this.blockPoolTokenSecretManager = new BlockPoolTokenSecretManager();
}
+ /**
+ * A data node is considered to be up if one of the bp services is up
+ */
public boolean isDatanodeUp() {
for (BPOfferService bp : blockPoolManager.getAllNamenodeThreads()) {
- // Datanode is up, if one of the bp service is up
if (bp.bpThread.isAlive()) {
return true;
}
@@ -2542,4 +2542,19 @@ public class DataNode extends Configured
}
return false;
}
+
+ /**
+ * A datanode is considered to be fully started if all the BP threads are
+ * alive and all the block pools are initialized.
+ *
+ * @return true - if the data node is fully started
+ */
+ public boolean isDatanodeFullyStarted() {
+ for (BPOfferService bp : blockPoolManager.getAllNamenodeThreads()) {
+ if (!bp.initialized() || !bp.shouldServiceRun || !bp.bpThread.isAlive()) {
+ return false;
+ }
+ }
+ return true;
+ }
}
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java Wed Mar 2 20:02:12 2011
@@ -220,8 +220,8 @@ public class DataStorage extends Storage
}
// mkdir for the list of BlockPoolStorage
makeBlockPoolDataDir(bpDataDirs, null);
- BlockPoolSliceStorage bpStorage = new BlockPoolSliceStorage(nsInfo.getNamespaceID(),
- bpID, nsInfo.getCTime(), nsInfo.getClusterID());
+ BlockPoolSliceStorage bpStorage = new BlockPoolSliceStorage(
+ nsInfo.getNamespaceID(), bpID, nsInfo.getCTime(), nsInfo.getClusterID());
bpStorage.recoverTransitionRead(nsInfo, bpDataDirs, startOpt);
addBlockPoolStorage(bpID, bpStorage);
@@ -278,6 +278,10 @@ public class DataStorage extends Storage
props.setProperty("cTime", String.valueOf(cTime));
props.setProperty("layoutVersion", String.valueOf(layoutVersion));
props.setProperty("storageID", storageID);
+ // Set NamespaceID in version LAST_PRE_FEDERATION_LAYOUT_VERSION or before
+ if (layoutVersion >= LAST_PRE_FEDERATION_LAYOUT_VERSION) {
+ props.setProperty("namespaceID", String.valueOf(namespaceID));
+ }
}
/*
@@ -358,12 +362,23 @@ public class DataStorage extends Storage
assert this.layoutVersion >= FSConstants.LAYOUT_VERSION :
"Future version is not allowed";
+ // For pre-federation version - validate the namespaceID
+ if (layoutVersion >= Storage.LAST_PRE_FEDERATION_LAYOUT_VERSION &&
+ getNamespaceID() != nsInfo.getNamespaceID()) {
+ throw new IOException("Incompatible namespaceIDs in "
+ + sd.getRoot().getCanonicalPath() + ": namenode namespaceID = "
+ + nsInfo.getNamespaceID() + "; datanode namespaceID = "
+ + getNamespaceID());
+ }
+
+ // For post federation version, validate clusterID
if (layoutVersion < Storage.LAST_PRE_FEDERATION_LAYOUT_VERSION
&& !getClusterID().equals(nsInfo.getClusterID())) {
throw new IOException("Incompatible clusterIDs in "
+ sd.getRoot().getCanonicalPath() + ": namenode clusterID = "
+ nsInfo.getClusterID() + "; datanode clusterID = " + getClusterID());
}
+
// regular start up
if (this.layoutVersion == FSConstants.LAYOUT_VERSION
&& this.cTime == nsInfo.getCTime())
@@ -421,6 +436,7 @@ public class DataStorage extends Storage
+ "; old CTime = " + this.getCTime()
+ ".\n new LV = " + nsInfo.getLayoutVersion()
+ "; new CTime = " + nsInfo.getCTime());
+
File curDir = sd.getCurrentDir();
File prevDir = sd.getPreviousDir();
assert curDir.exists() : "Data node current directory must exist.";
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Wed Mar 2 20:02:12 2011
@@ -233,8 +233,9 @@ public class FSImage extends Storage {
setStorageDirectories(fsDirs, fsEditsDirs);
}
- public FSImage(StorageInfo storageInfo) {
+ public FSImage(StorageInfo storageInfo, String bpid) {
super(NodeType.NAME_NODE, storageInfo);
+ blockpoolID = bpid;
}
/**
@@ -824,7 +825,10 @@ public class FSImage extends Storage {
StorageDirectory sd
) throws IOException {
super.setFields(props, sd);
- props.setProperty("blockpoolID", blockpoolID);
+ // Set blockpoolID in version LAST_PRE_FEDERATION_LAYOUT_VERSION or before
+ if (layoutVersion < LAST_PRE_FEDERATION_LAYOUT_VERSION) {
+ props.setProperty("blockpoolID", blockpoolID);
+ }
boolean uState = getDistributedUpgradeState();
int uVersion = getDistributedUpgradeVersion();
if(uState && uVersion != getLayoutVersion()) {
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/fs/TestFiRename.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/fs/TestFiRename.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/fs/TestFiRename.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/fs/TestFiRename.java Wed Mar 2 20:02:12 2011
@@ -87,7 +87,7 @@ public class TestFiRename {
cluster.shutdown();
cluster = null;
}
- cluster = new MiniDFSCluster(CONF, 1, format, null);
+ cluster = new MiniDFSCluster.Builder(CONF).format(format).build();
cluster.waitClusterUp();
fc = FileContext.getFileContext(cluster.getURI(0), CONF);
}
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java Wed Mar 2 20:02:12 2011
@@ -71,8 +71,8 @@ public class TestFiDataTransferProtocol
* 5. read the 1 byte and compare results
*/
private static void write1byte(String methodName) throws IOException {
- final MiniDFSCluster cluster = new MiniDFSCluster(conf, REPLICATION, true,
- null);
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(REPLICATION).format(true).build();
final FileSystem dfs = cluster.getFileSystem();
try {
final Path p = new Path("/" + methodName + "/foo");
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol2.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol2.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol2.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol2.java Wed Mar 2 20:02:12 2011
@@ -88,7 +88,8 @@ public class TestFiDataTransferProtocol2
FiTestUtil.LOG.info("size=" + size + ", nPackets=" + nPackets
+ ", lastPacketSize=" + lastPacketSize);
- final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION).build();
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(REPLICATION).build();
final FileSystem dfs = cluster.getFileSystem();
try {
final Path p = new Path("/" + methodName + "/foo");
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java Wed Mar 2 20:02:12 2011
@@ -62,8 +62,8 @@ public class TestFiPipelineClose {
* 5. read the 1 byte and compare results
*/
private static void write1byte(String methodName) throws IOException {
- final MiniDFSCluster cluster = new MiniDFSCluster(conf, REPLICATION, true,
- null);
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(REPLICATION).format(true).build();
final FileSystem dfs = cluster.getFileSystem();
try {
final Path p = new Path("/" + methodName + "/foo");
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java Wed Mar 2 20:02:12 2011
@@ -410,7 +410,6 @@ public class MiniDFSCluster {
StartupOption operation,
String[] racks, String hosts[],
long[] simulatedCapacities) throws IOException {
- this.numDataNodes = numDataNodes;
this.nameNodes = new NameNodeInfo[1]; // Single namenode in the cluster
initMiniDFSCluster(nameNodePort, conf, 1, format,
manageNameDfsDirs, manageDataDfsDirs, operation, racks, hosts,
@@ -775,7 +774,7 @@ public class MiniDFSCluster {
/**
* Finalize cluster for the namenode at the given index
- * @see {@link MiniDFSCluster#finalizeCluster(Configuration)}
+ * @see MiniDFSCluster#finalizeCluster(Configuration)
* @param nnIndex
* @param conf
* @throws Exception
@@ -1134,7 +1133,12 @@ public class MiniDFSCluster {
if (dataNodes == null || dataNodes.size() == 0) {
return false;
}
- return true;
+ for (DataNodeProperties dn : dataNodes) {
+ if (dn.datanode.isDatanodeUp()) {
+ return true;
+ }
+ }
+ return false;
}
/**
@@ -1255,6 +1259,15 @@ public class MiniDFSCluster {
if (dnInfo.length != numDataNodes) {
return true;
}
+
+ // check if all data nodes are fully started
+ for (DataNodeProperties dn : dataNodes) {
+ if (!dn.datanode.isDatanodeFullyStarted()) {
+ // if one of the data nodes is not fully started, continue to wait
+ return true;
+ }
+ }
+
// make sure all datanodes have sent first heartbeat to namenode,
// using (capacity == 0) as proxy.
for (DatanodeInfo dn : dnInfo) {
@@ -1262,6 +1275,13 @@ public class MiniDFSCluster {
return true;
}
}
+
+ // If datanode dataset is not initialized then wait
+ for (DataNodeProperties dn : dataNodes) {
+ if (dn.datanode.data == null) {
+ return true;
+ }
+ }
return false;
}
@@ -1400,7 +1420,7 @@ public class MiniDFSCluster {
* @return finalized directory for a block pool
*/
public static File getRbwDir(File storageDir, String bpid) {
- return new File(storageDir, "/current/" + bpid + "/rbw/");
+ return new File(storageDir, "/current/" + bpid + "/current/rbw/");
}
/**
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java Wed Mar 2 20:02:12 2011
@@ -125,6 +125,7 @@ public class TestDFSRollback extends Tes
File[] baseDirs;
UpgradeUtilities.initialize();
+ StorageInfo storageInfo = null;
for (int numDirs = 1; numDirs <= 2; numDirs++) {
conf = new HdfsConfiguration();
conf.setInt("dfs.datanode.scan.period.hours", -1);
@@ -192,11 +193,14 @@ public class TestDFSRollback extends Tes
.build();
UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "previous");
- UpgradeUtilities.createVersionFile(DATA_NODE, baseDirs,
- new StorageInfo(Integer.MIN_VALUE,
- UpgradeUtilities.getCurrentNamespaceID(cluster),
- UpgradeUtilities.getCurrentClusterID(cluster),
- UpgradeUtilities.getCurrentFsscTime(cluster)));
+ storageInfo = new StorageInfo(Integer.MIN_VALUE,
+ UpgradeUtilities.getCurrentNamespaceID(cluster),
+ UpgradeUtilities.getCurrentClusterID(cluster),
+ UpgradeUtilities.getCurrentFsscTime(cluster));
+
+ UpgradeUtilities.createDataNodeVersionFile(baseDirs, storageInfo,
+ UpgradeUtilities.getCurrentBlockPoolID(cluster));
+
startBlockPoolShouldFail(StartupOption.ROLLBACK,
cluster.getNamesystem().getBlockPoolId());
cluster.shutdown();
@@ -212,13 +216,16 @@ public class TestDFSRollback extends Tes
.manageNameDfsDirs(false)
.startupOption(StartupOption.ROLLBACK)
.build();
+
UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "previous");
- UpgradeUtilities.createVersionFile(DATA_NODE, baseDirs,
- new StorageInfo(UpgradeUtilities.getCurrentLayoutVersion(),
- UpgradeUtilities.getCurrentNamespaceID(cluster),
- UpgradeUtilities.getCurrentClusterID(cluster),
- Long.MAX_VALUE));
+ storageInfo = new StorageInfo(UpgradeUtilities.getCurrentLayoutVersion(),
+ UpgradeUtilities.getCurrentNamespaceID(cluster),
+ UpgradeUtilities.getCurrentClusterID(cluster), Long.MAX_VALUE);
+
+ UpgradeUtilities.createDataNodeVersionFile(baseDirs, storageInfo,
+ UpgradeUtilities.getCurrentBlockPoolID(cluster));
+
startBlockPoolShouldFail(StartupOption.ROLLBACK,
cluster.getNamesystem().getBlockPoolId());
cluster.shutdown();
@@ -255,11 +262,13 @@ public class TestDFSRollback extends Tes
log("NameNode rollback with old layout version in previous", numDirs);
UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
- UpgradeUtilities.createVersionFile(NAME_NODE, baseDirs,
- new StorageInfo(1,
- UpgradeUtilities.getCurrentNamespaceID(null),
- UpgradeUtilities.getCurrentClusterID(null),
- UpgradeUtilities.getCurrentFsscTime(null)));
+ storageInfo = new StorageInfo(1,
+ UpgradeUtilities.getCurrentNamespaceID(null),
+ UpgradeUtilities.getCurrentClusterID(null),
+ UpgradeUtilities.getCurrentFsscTime(null));
+
+ UpgradeUtilities.createNameNodeVersionFile(baseDirs,
+ storageInfo, UpgradeUtilities.getCurrentBlockPoolID(cluster));
startNameNodeShouldFail(StartupOption.UPGRADE);
UpgradeUtilities.createEmptyDirs(nameNodeDirs);
} // end numDir loop
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSStartupVersions.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSStartupVersions.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSStartupVersions.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSStartupVersions.java Wed Mar 2 20:02:12 2011
@@ -46,7 +46,8 @@ public class TestDFSStartupVersions exte
/**
* Writes an INFO log message containing the parameters.
*/
- void log(String label, NodeType nodeType, Integer testCase, StorageInfo version) {
+ void log(String label, NodeType nodeType, Integer testCase,
+ StorageData sd) {
String testCaseLine = "";
if (testCase != null) {
testCaseLine = " testCase="+testCase;
@@ -55,9 +56,26 @@ public class TestDFSStartupVersions exte
LOG.info("***TEST*** " + label + ":"
+ testCaseLine
+ " nodeType="+nodeType
- + " layoutVersion="+version.getLayoutVersion()
- + " namespaceID="+version.getNamespaceID()
- + " fsscTime="+version.getCTime());
+ + " layoutVersion="+sd.storageInfo.getLayoutVersion()
+ + " namespaceID="+sd.storageInfo.getNamespaceID()
+ + " fsscTime="+sd.storageInfo.getCTime()
+ + " clusterID="+sd.storageInfo.getClusterID()
+ + " BlockPoolID="+sd.blockPoolId);
+ }
+
+ /**
+ * Class used for initializing version information for tests
+ */
+ private static class StorageData {
+ private final StorageInfo storageInfo;
+ private final String blockPoolId;
+
+ StorageData(int layoutVersion, int namespaceId, String clusterId,
+ long cTime, String bpid) {
+ storageInfo = new StorageInfo(layoutVersion, namespaceId, clusterId,
+ cTime);
+ blockPoolId = bpid;
+ }
}
/**
@@ -67,7 +85,7 @@ public class TestDFSStartupVersions exte
* {currentNamespaceId,incorrectNamespaceId} X
* {pastFsscTime,currentFsscTime,futureFsscTime}
*/
- private StorageInfo[] initializeVersions() throws Exception {
+ private StorageData[] initializeVersions() throws Exception {
int layoutVersionOld = Storage.LAST_UPGRADABLE_LAYOUT_VERSION;
int layoutVersionCur = UpgradeUtilities.getCurrentLayoutVersion();
int layoutVersionNew = Integer.MIN_VALUE;
@@ -76,27 +94,54 @@ public class TestDFSStartupVersions exte
long fsscTimeOld = Long.MIN_VALUE;
long fsscTimeCur = UpgradeUtilities.getCurrentFsscTime(null);
long fsscTimeNew = Long.MAX_VALUE;
- String clusterID = "cid-test";
+ String clusterID = "testClusterID";
+ String invalidClusterID = "testClusterID";
+ String bpid = UpgradeUtilities.getCurrentBlockPoolID(null);
+ String invalidBpid = "invalidBpid";
- return new StorageInfo[] {
- new StorageInfo(layoutVersionOld, namespaceIdCur, clusterID, fsscTimeOld), // 0
- new StorageInfo(layoutVersionOld, namespaceIdCur, clusterID, fsscTimeCur), // 1
- new StorageInfo(layoutVersionOld, namespaceIdCur, clusterID, fsscTimeNew), // 2
- new StorageInfo(layoutVersionOld, namespaceIdOld, clusterID, fsscTimeOld), // 3
- new StorageInfo(layoutVersionOld, namespaceIdOld, clusterID, fsscTimeCur), // 4
- new StorageInfo(layoutVersionOld, namespaceIdOld, clusterID, fsscTimeNew), // 5
- new StorageInfo(layoutVersionCur, namespaceIdCur, clusterID, fsscTimeOld), // 6
- new StorageInfo(layoutVersionCur, namespaceIdCur, clusterID, fsscTimeCur), // 7
- new StorageInfo(layoutVersionCur, namespaceIdCur, clusterID, fsscTimeNew), // 8
- new StorageInfo(layoutVersionCur, namespaceIdOld, clusterID, fsscTimeOld), // 9
- new StorageInfo(layoutVersionCur, namespaceIdOld, clusterID, fsscTimeCur), // 10
- new StorageInfo(layoutVersionCur, namespaceIdOld, clusterID, fsscTimeNew), // 11
- new StorageInfo(layoutVersionNew, namespaceIdCur, clusterID, fsscTimeOld), // 12
- new StorageInfo(layoutVersionNew, namespaceIdCur, clusterID, fsscTimeCur), // 13
- new StorageInfo(layoutVersionNew, namespaceIdCur, clusterID, fsscTimeNew), // 14
- new StorageInfo(layoutVersionNew, namespaceIdOld, clusterID, fsscTimeOld), // 15
- new StorageInfo(layoutVersionNew, namespaceIdOld, clusterID, fsscTimeCur), // 16
- new StorageInfo(layoutVersionNew, namespaceIdOld, clusterID, fsscTimeNew), // 17
+ return new StorageData[] {
+ new StorageData(layoutVersionOld, namespaceIdCur, clusterID,
+ fsscTimeOld, bpid), // 0
+ new StorageData(layoutVersionOld, namespaceIdCur, clusterID,
+ fsscTimeCur, bpid), // 1
+ new StorageData(layoutVersionOld, namespaceIdCur, clusterID,
+ fsscTimeNew, bpid), // 2
+ new StorageData(layoutVersionOld, namespaceIdOld, clusterID,
+ fsscTimeOld, bpid), // 3
+ new StorageData(layoutVersionOld, namespaceIdOld, clusterID,
+ fsscTimeCur, bpid), // 4
+ new StorageData(layoutVersionOld, namespaceIdOld, clusterID,
+ fsscTimeNew, bpid), // 5
+ new StorageData(layoutVersionCur, namespaceIdCur, clusterID,
+ fsscTimeOld, bpid), // 6
+ new StorageData(layoutVersionCur, namespaceIdCur, clusterID,
+ fsscTimeCur, bpid), // 7
+ new StorageData(layoutVersionCur, namespaceIdCur, clusterID,
+ fsscTimeNew, bpid), // 8
+ new StorageData(layoutVersionCur, namespaceIdOld, clusterID,
+ fsscTimeOld, bpid), // 9
+ new StorageData(layoutVersionCur, namespaceIdOld, clusterID,
+ fsscTimeCur, bpid), // 10
+ new StorageData(layoutVersionCur, namespaceIdOld, clusterID,
+ fsscTimeNew, bpid), // 11
+ new StorageData(layoutVersionNew, namespaceIdCur, clusterID,
+ fsscTimeOld, bpid), // 12
+ new StorageData(layoutVersionNew, namespaceIdCur, clusterID,
+ fsscTimeCur, bpid), // 13
+ new StorageData(layoutVersionNew, namespaceIdCur, clusterID,
+ fsscTimeNew, bpid), // 14
+ new StorageData(layoutVersionNew, namespaceIdOld, clusterID,
+ fsscTimeOld, bpid), // 15
+ new StorageData(layoutVersionNew, namespaceIdOld, clusterID,
+ fsscTimeCur, bpid), // 16
+ new StorageData(layoutVersionNew, namespaceIdOld, clusterID,
+ fsscTimeNew, bpid), // 17
+ // Test with invalid clusterId
+ new StorageData(layoutVersionCur, namespaceIdCur, invalidClusterID,
+ fsscTimeCur, bpid), // 18
+ // Test with invalid block pool Id
+ new StorageData(layoutVersionCur, namespaceIdCur, clusterID,
+ fsscTimeCur, invalidBpid) // 19
};
}
@@ -107,29 +152,52 @@ public class TestDFSStartupVersions exte
* will work together. The rules for compatibility,
* taken from the DFS Upgrade Design, are as follows:
* <pre>
- * 1. The data-node does regular startup (no matter which options
+ * <ol>
+ * <li>Check 0: Datanode namespaceID != Namenode namespaceID the startup fails
+ * </li>
+ * <li>Check 1: Datanode clusterID != Namenode clusterID the startup fails
+ * </li>
+ * <li>Check 2: Datanode blockPoolID != Namenode blockPoolID the startup fails
+ * </li>
+ * <li>Check 3: The data-node does regular startup (no matter which options
* it is started with) if
* softwareLV == storedLV AND
* DataNode.FSSCTime == NameNode.FSSCTime
- * 2. The data-node performs an upgrade if it is started without any
+ * </li>
+ * <li>Check 4: The data-node performs an upgrade if it is started without any
* options and
* |softwareLV| > |storedLV| OR
* (softwareLV == storedLV AND
* DataNode.FSSCTime < NameNode.FSSCTime)
- * 3. NOT TESTED: The data-node rolls back if it is started with
+ * </li>
+ * <li>NOT TESTED: The data-node rolls back if it is started with
* the -rollback option and
* |softwareLV| >= |previous.storedLV| AND
* DataNode.previous.FSSCTime <= NameNode.FSSCTime
- * 4. In all other cases the startup fails.
+ * </li>
+ * <li>Check 5: In all other cases the startup fails.</li>
+ * </ol>
* </pre>
*/
- boolean isVersionCompatible(StorageInfo namenodeVer, StorageInfo datanodeVer) {
+ boolean isVersionCompatible(StorageData namenodeSd, StorageData datanodeSd) {
+ final StorageInfo namenodeVer = namenodeSd.storageInfo;
+ final StorageInfo datanodeVer = datanodeSd.storageInfo;
// check #0
if (namenodeVer.getNamespaceID() != datanodeVer.getNamespaceID()) {
LOG.info("namespaceIDs are not equal: isVersionCompatible=false");
return false;
}
// check #1
+ if (!namenodeVer.getClusterID().equals(datanodeVer.getClusterID())) {
+ LOG.info("clusterIDs are not equal: isVersionCompatible=false");
+ return false;
+ }
+ // check #2
+ if (!namenodeSd.blockPoolId.equals(datanodeSd.blockPoolId)) {
+ LOG.info("blockPoolIDs are not equal: isVersionCompatible=false");
+ return false;
+ }
+ // check #3
int softwareLV = FSConstants.LAYOUT_VERSION; // will also be Namenode's LV
int storedLV = datanodeVer.getLayoutVersion();
if (softwareLV == storedLV &&
@@ -138,7 +206,7 @@ public class TestDFSStartupVersions exte
LOG.info("layoutVersions and cTimes are equal: isVersionCompatible=true");
return true;
}
- // check #2
+ // check #4
long absSoftwareLV = Math.abs((long)softwareLV);
long absStoredLV = Math.abs((long)storedLV);
if (absSoftwareLV > absStoredLV ||
@@ -148,7 +216,7 @@ public class TestDFSStartupVersions exte
LOG.info("softwareLayoutVersion is newer OR namenode cTime is newer: isVersionCompatible=true");
return true;
}
- // check #4
+ // check #5
LOG.info("default case: isVersionCompatible=false");
return false;
}
@@ -171,7 +239,7 @@ public class TestDFSStartupVersions exte
UpgradeUtilities.initialize();
Configuration conf = UpgradeUtilities.initializeStorageStateConf(1,
new HdfsConfiguration());
- StorageInfo[] versions = initializeVersions();
+ StorageData[] versions = initializeVersions();
UpgradeUtilities.createStorageDirs(
NAME_NODE, conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY), "current");
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
@@ -180,17 +248,21 @@ public class TestDFSStartupVersions exte
.manageNameDfsDirs(false)
.startupOption(StartupOption.REGULAR)
.build();
- StorageInfo nameNodeVersion = new StorageInfo(
- UpgradeUtilities.getCurrentLayoutVersion(),
- UpgradeUtilities.getCurrentNamespaceID(cluster),
- UpgradeUtilities.getCurrentClusterID(cluster),
- UpgradeUtilities.getCurrentFsscTime(cluster));
+ StorageData nameNodeVersion = new StorageData(
+ UpgradeUtilities.getCurrentLayoutVersion(),
+ UpgradeUtilities.getCurrentNamespaceID(cluster),
+ UpgradeUtilities.getCurrentClusterID(cluster),
+ UpgradeUtilities.getCurrentFsscTime(cluster),
+ UpgradeUtilities.getCurrentBlockPoolID(cluster));
+
log("NameNode version info", NAME_NODE, null, nameNodeVersion);
+ String bpid = UpgradeUtilities.getCurrentBlockPoolID(cluster);
for (int i = 0; i < versions.length; i++) {
- File[] storage = UpgradeUtilities.createStorageDirs(
- DATA_NODE, conf.getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY), "current");
+ File[] storage = UpgradeUtilities.createStorageDirs(DATA_NODE,
+ conf.getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY), "current");
log("DataNode version info", DATA_NODE, i, versions[i]);
- UpgradeUtilities.createVersionFile(DATA_NODE, storage, versions[i]);
+ UpgradeUtilities.createDataNodeVersionFile(storage,
+ versions[i].storageInfo, bpid, versions[i].blockPoolId);
try {
cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null);
} catch (Exception ignore) {
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java Wed Mar 2 20:02:12 2011
@@ -147,6 +147,7 @@ public class TestDFSUpgrade extends Test
File[] baseDirs;
UpgradeUtilities.initialize();
+ StorageInfo storageInfo = null;
for (int numDirs = 1; numDirs <= 2; numDirs++) {
conf = new HdfsConfiguration();
conf.setInt("dfs.datanode.scan.period.hours", -1);
@@ -192,11 +193,14 @@ public class TestDFSUpgrade extends Test
UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
cluster = createCluster();
baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
- UpgradeUtilities.createVersionFile(DATA_NODE, baseDirs,
- new StorageInfo(Integer.MIN_VALUE,
- UpgradeUtilities.getCurrentNamespaceID(cluster),
- UpgradeUtilities.getCurrentClusterID(cluster),
- UpgradeUtilities.getCurrentFsscTime(cluster)));
+ storageInfo = new StorageInfo(Integer.MIN_VALUE,
+ UpgradeUtilities.getCurrentNamespaceID(cluster),
+ UpgradeUtilities.getCurrentClusterID(cluster),
+ UpgradeUtilities.getCurrentFsscTime(cluster));
+
+ UpgradeUtilities.createDataNodeVersionFile(baseDirs, storageInfo,
+ UpgradeUtilities.getCurrentBlockPoolID(cluster));
+
startBlockPoolShouldFail(StartupOption.REGULAR, UpgradeUtilities
.getCurrentBlockPoolID(null));
cluster.shutdown();
@@ -207,11 +211,12 @@ public class TestDFSUpgrade extends Test
UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
cluster = createCluster();
baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
- UpgradeUtilities.createVersionFile(DATA_NODE, baseDirs,
- new StorageInfo(UpgradeUtilities.getCurrentLayoutVersion(),
- UpgradeUtilities.getCurrentNamespaceID(cluster),
- UpgradeUtilities.getCurrentClusterID(cluster),
- Long.MAX_VALUE));
+ storageInfo = new StorageInfo(UpgradeUtilities.getCurrentLayoutVersion(),
+ UpgradeUtilities.getCurrentNamespaceID(cluster),
+ UpgradeUtilities.getCurrentClusterID(cluster), Long.MAX_VALUE);
+
+ UpgradeUtilities.createDataNodeVersionFile(baseDirs, storageInfo,
+ UpgradeUtilities.getCurrentBlockPoolID(cluster));
// Ensure corresponding block pool failed to initialized
startBlockPoolShouldFail(StartupOption.REGULAR, UpgradeUtilities
.getCurrentBlockPoolID(null));
@@ -245,21 +250,27 @@ public class TestDFSUpgrade extends Test
log("NameNode upgrade with old layout version in current", numDirs);
baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
- UpgradeUtilities.createVersionFile(NAME_NODE, baseDirs,
- new StorageInfo(Storage.LAST_UPGRADABLE_LAYOUT_VERSION + 1,
- UpgradeUtilities.getCurrentNamespaceID(null),
- UpgradeUtilities.getCurrentClusterID(null),
- UpgradeUtilities.getCurrentFsscTime(null)));
+ storageInfo = new StorageInfo(Storage.LAST_UPGRADABLE_LAYOUT_VERSION + 1,
+ UpgradeUtilities.getCurrentNamespaceID(null),
+ UpgradeUtilities.getCurrentClusterID(null),
+ UpgradeUtilities.getCurrentFsscTime(null));
+
+ UpgradeUtilities.createNameNodeVersionFile(baseDirs, storageInfo,
+ UpgradeUtilities.getCurrentBlockPoolID(cluster));
+
startNameNodeShouldFail(StartupOption.UPGRADE);
UpgradeUtilities.createEmptyDirs(nameNodeDirs);
log("NameNode upgrade with future layout version in current", numDirs);
baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
- UpgradeUtilities.createVersionFile(NAME_NODE, baseDirs,
- new StorageInfo(Integer.MIN_VALUE,
- UpgradeUtilities.getCurrentNamespaceID(null),
- UpgradeUtilities.getCurrentClusterID(null),
- UpgradeUtilities.getCurrentFsscTime(null)));
+ storageInfo = new StorageInfo(Integer.MIN_VALUE,
+ UpgradeUtilities.getCurrentNamespaceID(null),
+ UpgradeUtilities.getCurrentClusterID(null),
+ UpgradeUtilities.getCurrentFsscTime(null));
+
+ UpgradeUtilities.createNameNodeVersionFile(baseDirs, storageInfo,
+ UpgradeUtilities.getCurrentBlockPoolID(cluster));
+
startNameNodeShouldFail(StartupOption.UPGRADE);
UpgradeUtilities.createEmptyDirs(nameNodeDirs);
} // end numDir loop
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java Wed Mar 2 20:02:12 2011
@@ -44,10 +44,10 @@ import static org.apache.hadoop.hdfs.ser
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage;
import org.apache.hadoop.hdfs.server.datanode.DataStorage;
import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
import org.apache.hadoop.test.GenericTestUtils;
/**
@@ -313,31 +313,25 @@ public class UpgradeUtilities {
}
/**
- * Create a <code>version</code> file inside the specified parent
+ * Create a <code>version</code> file for namenode inside the specified parent
* directory. If such a file already exists, it will be overwritten.
* The given version string will be written to the file as the layout
* version. None of the parameters may be null.
*
- * @param version
+ * @param parent directory where namenode VERSION file is stored
+ * @param version StorageInfo to create VERSION file from
+ * @param bpid Block pool Id
*
* @return the created version file
*/
- public static File[] createVersionFile(NodeType nodeType, File[] parent,
- StorageInfo version) throws IOException
- {
- Storage storage = null;
+ public static File[] createNameNodeVersionFile(File[] parent,
+ StorageInfo version, String bpid) throws IOException {
+ FSImage storage = null;
File[] versionFiles = new File[parent.length];
for (int i = 0; i < parent.length; i++) {
File versionFile = new File(parent[i], "VERSION");
FileUtil.fullyDelete(versionFile);
- switch (nodeType) {
- case NAME_NODE:
- storage = new FSImage(version);
- break;
- case DATA_NODE:
- storage = new DataStorage(version, "doNotCare");
- break;
- }
+ storage = new FSImage(version, bpid);
StorageDirectory sd = storage.new StorageDirectory(parent[i].getParentFile());
sd.write(versionFile);
versionFiles[i] = versionFile;
@@ -346,6 +340,62 @@ public class UpgradeUtilities {
}
/**
+ * Create a <code>version</code> file for datanode inside the specified parent
+ * directory. If such a file already exists, it will be overwritten.
+ * The given version string will be written to the file as the layout
+ * version. None of the parameters may be null.
+ *
+ * @param parent directory where namenode VERSION file is stored
+ * @param version StorageInfo to create VERSION file from
+ * @param bpid Block pool Id
+ */
+ public static void createDataNodeVersionFile(File[] parent,
+ StorageInfo version, String bpid) throws IOException {
+ createDataNodeVersionFile(parent, version, bpid, bpid);
+ }
+
+ /**
+ * Create a <code>version</code> file for datanode inside the specified parent
+ * directory. If such a file already exists, it will be overwritten.
+ * The given version string will be written to the file as the layout
+ * version. None of the parameters may be null.
+ *
+ * @param parent directory where namenode VERSION file is stored
+ * @param version StorageInfo to create VERSION file from
+ * @param bpid Block pool Id
+ * @param bpidToWrite Block pool Id to write into the version file
+ */
+ public static void createDataNodeVersionFile(File[] parent,
+ StorageInfo version, String bpid, String bpidToWrite) throws IOException {
+ DataStorage storage = null;
+ File[] versionFiles = new File[parent.length];
+ for (int i = 0; i < parent.length; i++) {
+ File versionFile = new File(parent[i], "VERSION");
+ FileUtil.fullyDelete(versionFile);
+ storage = new DataStorage(version, "doNotCare");
+ StorageDirectory sd = storage.new StorageDirectory(parent[i].getParentFile());
+ sd.write(versionFile);
+ versionFiles[i] = versionFile;
+ File bpDir = BlockPoolSliceStorage.getBpRoot(bpid, parent[i]);
+ createBlockPoolVersionFile(bpDir, version, bpidToWrite);
+ }
+ }
+
+ public static void createBlockPoolVersionFile(File bpDir,
+ StorageInfo version, String bpid) throws IOException {
+ // Create block pool version files
+ if (version.layoutVersion < Storage.LAST_PRE_FEDERATION_LAYOUT_VERSION) {
+ File bpCurDir = new File(bpDir, Storage.STORAGE_DIR_CURRENT);
+ BlockPoolSliceStorage bpStorage = new BlockPoolSliceStorage(version,
+ bpid);
+ File versionFile = new File(bpCurDir, "VERSION");
+ FileUtil.fullyDelete(versionFile);
+ StorageDirectory sd = bpStorage.new StorageDirectory(bpDir);
+ sd.write(versionFile);
+ }
+ }
+
+ /**
* Corrupt the specified file. Some random bytes within the file
* will be changed to some random values.
*
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestMulitipleNNDataBlockScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestMulitipleNNDataBlockScanner.java?rev=1076358&r1=1076357&r2=1076358&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestMulitipleNNDataBlockScanner.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestMulitipleNNDataBlockScanner.java Wed Mar 2 20:02:12 2011
@@ -140,10 +140,8 @@ public class TestMulitipleNNDataBlockSca
setUp(9943);
try {
cluster.restartDataNode(0);
+ cluster.waitActive();
DataNode dn = cluster.getDataNodes().get(0);
- while (!dn.isDatanodeUp()) {
- Thread.sleep(2000);
- }
for (int i = 0; i < 3; i++) {
long blocksScanned = 0;
while (blocksScanned != 20) {