You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dh...@apache.org on 2008/03/15 00:59:56 UTC
svn commit: r637307 - in /hadoop/core/trunk: ./
src/java/org/apache/hadoop/dfs/ src/test/org/apache/hadoop/dfs/
Author: dhruba
Date: Fri Mar 14 16:59:54 2008
New Revision: 637307
URL: http://svn.apache.org/viewvc?rev=637307&view=rev
Log:
HADOOP-2992. Distributed Upgrade framework works correctly with
more than one upgrade object. (Konstantin Shvachko via dhruba)
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerDatanode.java
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerNamenode.java
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeObject.java
hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java
hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDistributedUpgrade.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=637307&r1=637306&r2=637307&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri Mar 14 16:59:54 2008
@@ -222,6 +222,9 @@
minidfscluster sometimes creates datanodes with ports that are
different from their original instance. (dhruba)
+ HADOOP-2992. Distributed Upgrade framework works correctly with
+ more than one upgrade object. (Konstantin Shvachko via dhruba)
+
Release 0.16.1 - 2008-03-13
INCOMPATIBLE CHANGES
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerDatanode.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerDatanode.java?rev=637307&r1=637306&r2=637307&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerDatanode.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerDatanode.java Fri Mar 14 16:59:54 2008
@@ -47,8 +47,9 @@
void initializeUpgrade(NamespaceInfo nsInfo) throws IOException {
if( ! super.initializeUpgrade())
return; // distr upgrade is not needed
- DataNode.LOG.info("\n Distributed upgrade for DataNode version "
- + getUpgradeVersion() + " to current LV "
+ DataNode.LOG.info("\n Distributed upgrade for DataNode "
+ + dataNode.dnRegistration.getName()
+ + " version " + getUpgradeVersion() + " to current LV "
+ FSConstants.LAYOUT_VERSION + " is initialized.");
UpgradeObjectDatanode curUO = (UpgradeObjectDatanode)currentUpgrades.first();
curUO.setDatanode(dataNode);
@@ -72,15 +73,21 @@
return true;
}
if(broadcastCommand != null) {
- // the upgrade has been finished by this data-node,
- // but the cluster is still running it,
- // reply with the broadcast command
- assert currentUpgrades == null :
- "UpgradeManagerDatanode.currentUpgrades is not null.";
- assert upgradeDaemon == null :
- "UpgradeManagerDatanode.upgradeDaemon is not null.";
- dataNode.namenode.processUpgradeCommand(broadcastCommand);
- return true;
+ if(broadcastCommand.getVersion() > this.getUpgradeVersion()) {
+ // stop broadcasting, the cluster moved on
+ // start upgrade for the next version
+ broadcastCommand = null;
+ } else {
+ // the upgrade has been finished by this data-node,
+ // but the cluster is still running it,
+ // reply with the broadcast command
+ assert currentUpgrades == null :
+ "UpgradeManagerDatanode.currentUpgrades is not null.";
+ assert upgradeDaemon == null :
+ "UpgradeManagerDatanode.upgradeDaemon is not null.";
+ dataNode.namenode.processUpgradeCommand(broadcastCommand);
+ return true;
+ }
}
if(currentUpgrades == null)
currentUpgrades = getDistributedUpgrades();
@@ -91,18 +98,15 @@
+ "The upgrade object is not defined.");
return false;
}
- if(currentUpgrades.size() > 1)
- throw new IOException(
- "More than one distributed upgrade objects registered for version "
- + getUpgradeVersion());
upgradeState = true;
UpgradeObjectDatanode curUO = (UpgradeObjectDatanode)currentUpgrades.first();
curUO.setDatanode(dataNode);
curUO.startUpgrade();
upgradeDaemon = new Daemon(curUO);
upgradeDaemon.start();
- DataNode.LOG.info("\n Distributed upgrade for DataNode version "
- + getUpgradeVersion() + " to current LV "
+ DataNode.LOG.info("\n Distributed upgrade for DataNode "
+ + dataNode.dnRegistration.getName()
+ + " version " + getUpgradeVersion() + " to current LV "
+ FSConstants.LAYOUT_VERSION + " is started.");
return true;
}
@@ -116,8 +120,8 @@
if(startUpgrade()) // upgrade started
return;
throw new IOException(
- "Distributed upgrade for DataNode version "
- + getUpgradeVersion() + " to current LV "
+ "Distributed upgrade for DataNode " + dataNode.dnRegistration.getName()
+ + " version " + getUpgradeVersion() + " to current LV "
+ FSConstants.LAYOUT_VERSION + " cannot be started. "
+ "The upgrade object is not defined.");
}
@@ -130,8 +134,9 @@
upgradeState = false;
currentUpgrades = null;
upgradeDaemon = null;
- DataNode.LOG.info("\n Distributed upgrade for DataNode version "
- + getUpgradeVersion() + " to current LV "
+ DataNode.LOG.info("\n Distributed upgrade for DataNode "
+ + dataNode.dnRegistration.getName()
+ + " version " + getUpgradeVersion() + " to current LV "
+ FSConstants.LAYOUT_VERSION + " is complete.");
}
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerNamenode.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerNamenode.java?rev=637307&r1=637306&r2=637307&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerNamenode.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeManagerNamenode.java Fri Mar 14 16:59:54 2008
@@ -81,6 +81,9 @@
}
// current upgrade is done
curUO.completeUpgrade();
+ NameNode.LOG.info("\n Distributed upgrade for NameNode version "
+ + curUO.getVersion() + " to current LV "
+ + FSConstants.LAYOUT_VERSION + " is complete.");
// proceede with the next one
currentUpgrades.remove(curUO);
if(currentUpgrades.isEmpty()) { // all upgrades are done
@@ -93,9 +96,6 @@
}
synchronized void completeUpgrade() throws IOException {
- NameNode.LOG.info("\n Distributed upgrade for NameNode version "
- + getUpgradeVersion() + " to current LV "
- + FSConstants.LAYOUT_VERSION + " is complete.");
// set and write new upgrade state into disk
setUpgradeState(false, FSConstants.LAYOUT_VERSION);
FSNamesystem.getFSNamesystem().getFSImage().writeAll();
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeObject.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeObject.java?rev=637307&r1=637306&r2=637307&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeObject.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/UpgradeObject.java Fri Mar 14 16:59:54 2008
@@ -45,7 +45,7 @@
public int compareTo(Upgradeable o) {
if(this.getVersion() != o.getVersion())
- return (getVersion() < o.getVersion() ? -1 : 1);
+ return (getVersion() > o.getVersion() ? -1 : 1);
int res = this.getType().toString().compareTo(o.getType().toString());
if(res != 0)
return res;
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java?rev=637307&r1=637306&r2=637307&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSUpgradeFromImage.java Fri Mar 14 16:59:54 2008
@@ -30,7 +30,6 @@
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.io.UTF8;
import org.apache.hadoop.dfs.FSConstants.StartupOption;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -61,6 +60,10 @@
boolean printChecksum = false;
protected void setUp() throws IOException {
+ unpackStorage();
+ }
+
+ void unpackStorage() throws IOException {
String tarFile = System.getProperty("test.cache.data") +
"/hadoop-12-dfs-dir.tgz";
String dataDir = System.getProperty("test.build.data");
@@ -178,13 +181,12 @@
MiniDFSCluster cluster = null;
try {
Configuration conf = new Configuration();
- cluster = new MiniDFSCluster(0, conf, numDataNodes, false,
- true, StartupOption.UPGRADE,
- null);
+ conf.setInt("dfs.datanode.scan.period.hours", -1); // block scanning off
+ cluster = new MiniDFSCluster(0, conf, numDataNodes, false, true,
+ StartupOption.UPGRADE, null);
cluster.waitActive();
DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost",
- cluster.getNameNodePort()),
- conf);
+ cluster.getNameNodePort()), conf);
//Safemode will be off only after upgrade is complete. Wait for it.
while ( dfsClient.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_GET) ) {
LOG.info("Waiting for SafeMode to be OFF.");
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDistributedUpgrade.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDistributedUpgrade.java?rev=637307&r1=637306&r2=637307&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDistributedUpgrade.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDistributedUpgrade.java Fri Mar 14 16:59:54 2008
@@ -17,15 +17,12 @@
*/
package org.apache.hadoop.dfs;
-import java.io.File;
import java.io.IOException;
import junit.framework.TestCase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import static org.apache.hadoop.dfs.FSConstants.NodeType.DATA_NODE;
-import static org.apache.hadoop.dfs.FSConstants.NodeType.NAME_NODE;
import static org.apache.hadoop.dfs.FSConstants.LAYOUT_VERSION;
import org.apache.hadoop.dfs.FSConstants.StartupOption;
@@ -79,33 +76,30 @@
/**
*/
public void testDistributedUpgrade() throws Exception {
- File[] baseDirs;
int numDirs = 1;
- UpgradeUtilities.initialize();
+ TestDFSUpgradeFromImage testImg = new TestDFSUpgradeFromImage();
+ testImg.unpackStorage();
+ int numDNs = testImg.numDataNodes;
// register new upgrade objects (ignore all existing)
UpgradeObjectCollection.initialize();
- UpgradeObjectCollection.registerUpgrade(new UpgradeObject_Test_Datanode());
- UpgradeObjectCollection.registerUpgrade(new UpgradeObject_Test_Namenode());
+ UpgradeObjectCollection.registerUpgrade(new UO_Datanode1());
+ UpgradeObjectCollection.registerUpgrade(new UO_Namenode1());
+ UpgradeObjectCollection.registerUpgrade(new UO_Datanode2());
+ UpgradeObjectCollection.registerUpgrade(new UO_Namenode2());
+ UpgradeObjectCollection.registerUpgrade(new UO_Datanode3());
+ UpgradeObjectCollection.registerUpgrade(new UO_Namenode3());
- conf = UpgradeUtilities.initializeStorageStateConf(numDirs,
- new Configuration());
- String[] nameNodeDirs = conf.getStrings("dfs.name.dir");
- String[] dataNodeDirs = conf.getStrings("dfs.data.dir");
- DFSAdmin dfsAdmin = new DFSAdmin();
- dfsAdmin.setConf(conf);
- String[] pars = {"-safemode", "wait"};
+ conf = new Configuration();
+ conf.setInt("dfs.datanode.scan.period.hours", -1); // block scanning off
log("NameNode start in regular mode when dustributed upgrade is required", numDirs);
- baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
- UpgradeUtilities.createVersionFile(NAME_NODE, baseDirs,
- new StorageInfo(LAYOUT_VERSION+2,
- UpgradeUtilities.getCurrentNamespaceID(cluster),
- UpgradeUtilities.getCurrentFsscTime(cluster)));
startNameNodeShouldFail(StartupOption.REGULAR);
log("Start NameNode only distributed upgrade", numDirs);
- cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
+ // cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
+ cluster = new MiniDFSCluster(0, conf, 0, false, true,
+ StartupOption.UPGRADE, null);
cluster.shutdown();
log("NameNode start in regular mode when dustributed upgrade has been started", numDirs);
@@ -115,19 +109,19 @@
startNameNodeShouldFail(StartupOption.ROLLBACK);
log("Normal distributed upgrade for the cluster", numDirs);
- cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
- UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
- cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null);
- dfsAdmin.run(pars);
+ cluster = new MiniDFSCluster(0, conf, numDNs, false, true,
+ StartupOption.UPGRADE, null);
+ DFSAdmin dfsAdmin = new DFSAdmin();
+ dfsAdmin.setConf(conf);
+ dfsAdmin.run(new String[] {"-safemode", "wait"});
cluster.shutdown();
// it should be ok to start in regular mode
log("NameCluster regular startup after the upgrade", numDirs);
- cluster = new MiniDFSCluster(conf, 0, StartupOption.REGULAR);
- cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null);
+ cluster = new MiniDFSCluster(0, conf, numDNs, false, true,
+ StartupOption.REGULAR, null);
+ cluster.waitActive();
cluster.shutdown();
- UpgradeUtilities.createEmptyDirs(nameNodeDirs);
- UpgradeUtilities.createEmptyDirs(dataNodeDirs);
}
public static void main(String[] args) throws Exception {
@@ -139,9 +133,16 @@
/**
* Upgrade object for data-node
*/
-class UpgradeObject_Test_Datanode extends UpgradeObjectDatanode {
+class UO_Datanode extends UpgradeObjectDatanode {
+ int version;
+
+ UO_Datanode(int v) {
+ this.status = (short)0;
+ version = v;
+ }
+
public int getVersion() {
- return LAYOUT_VERSION+1;
+ return version;
}
public void doUpgrade() throws IOException {
@@ -152,7 +153,6 @@
}
public UpgradeCommand startUpgrade() throws IOException {
- this.status = (short)0;
return null;
}
}
@@ -160,16 +160,23 @@
/**
* Upgrade object for name-node
*/
-class UpgradeObject_Test_Namenode extends UpgradeObjectNamenode {
+class UO_Namenode extends UpgradeObjectNamenode {
+ int version;
+
+ UO_Namenode(int v) {
+ status = (short)0;
+ version = v;
+ }
+
public int getVersion() {
- return LAYOUT_VERSION+1;
+ return version;
}
synchronized public UpgradeCommand processUpgradeCommand(
UpgradeCommand command) throws IOException {
switch(command.action) {
case UpgradeCommand.UC_ACTION_REPORT_STATUS:
- this.status += command.getCurrentStatus()/2; // 2 reports needed
+ this.status += command.getCurrentStatus()/8; // 4 reports needed
break;
default:
this.status++;
@@ -179,5 +186,41 @@
public UpgradeCommand completeUpgrade() throws IOException {
return null;
+ }
+}
+
+class UO_Datanode1 extends UO_Datanode {
+ UO_Datanode1() {
+ super(LAYOUT_VERSION+1);
+ }
+}
+
+class UO_Namenode1 extends UO_Namenode {
+ UO_Namenode1() {
+ super(LAYOUT_VERSION+1);
+ }
+}
+
+class UO_Datanode2 extends UO_Datanode {
+ UO_Datanode2() {
+ super(LAYOUT_VERSION+2);
+ }
+}
+
+class UO_Namenode2 extends UO_Namenode {
+ UO_Namenode2() {
+ super(LAYOUT_VERSION+2);
+ }
+}
+
+class UO_Datanode3 extends UO_Datanode {
+ UO_Datanode3() {
+ super(LAYOUT_VERSION+3);
+ }
+}
+
+class UO_Namenode3 extends UO_Namenode {
+ UO_Namenode3() {
+ super(LAYOUT_VERSION+3);
}
}