You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2012/02/13 22:00:41 UTC
svn commit: r1243691 - in
/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project:
hadoop-hdfs-httpfs/src/site/ hadoop-hdfs/
hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/
hadoop-hdfs/src/main/java/ hadoop-hdfs/src/main/java/org/apache/hadoop...
Author: todd
Date: Mon Feb 13 21:00:37 2012
New Revision: 1243691
URL: http://svn.apache.org/viewvc?rev=1243691&view=rev
Log:
Merge trunk into HA branch.
Some conflicts around TestBlockRecovery: mocking changed for new heartbeat types/responses
Added:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
- copied, changed from r1243679, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
Removed:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/unit/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/site.xml
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/webhdfs.xml
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockVolumeChoosingPolicy.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 13 21:00:37 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:1152502-1242870
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:1152502-1243679
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-1052:987665-1095512
/hadoop/hdfs/branches/HDFS-265:796829-820463
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/site.xml?rev=1243691&r1=1243690&r2=1243691&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/site.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/site.xml Mon Feb 13 21:00:37 2012
@@ -14,21 +14,16 @@
-->
<project name="HttpFS">
- <version position="right"/>
+ <skin>
+ <groupId>org.apache.maven.skins</groupId>
+ <artifactId>maven-stylus-skin</artifactId>
+ <version>1.2</version>
+ </skin>
- <bannerLeft>
- <name> </name>
- </bannerLeft>
-
- <skin>
- <groupId>org.apache.maven.skins</groupId>
- <artifactId>maven-stylus-skin</artifactId>
- <version>1.2</version>
- </skin>
-
- <body>
- <links>
- </links>
- </body>
+ <body>
+ <links>
+ <item name="Apache Hadoop" href="http://hadoop.apache.org/"/>
+ </links>
+ </body>
</project>
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1243691&r1=1243690&r2=1243691&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Feb 13 21:00:37 2012
@@ -129,6 +129,9 @@ Trunk (unreleased changes)
HDFS-2486. Remove unnecessary priority level checks in
UnderReplicatedBlocks. (Uma Maheswara Rao G via szetszwo)
+ HDFS-2878. Fix TestBlockRecovery and move it back into main test directory.
+ (todd)
+
OPTIMIZATIONS
HDFS-2477. Optimize computing the diff between a block report and the
namenode state. (Tomasz Nykiel via hairong)
@@ -216,6 +219,9 @@ Release 0.23.2 - UNRELEASED
IMPROVEMENTS
+ HDFS-2931. Switch DataNode's BlockVolumeChoosingPolicy to private-audience.
+ (harsh via szetszwo)
+
OPTIMIZATIONS
BUG FIXES
@@ -224,6 +230,14 @@ Release 0.23.2 - UNRELEASED
HDFS-2764. TestBackupNode is racy. (atm)
+ HDFS-2869. Fix an error in the webhdfs docs for the mkdir op (harsh)
+
+ HDFS-776. Fix exception handling in Balancer. (Uma Maheswara Rao G
+ via szetszwo)
+
+ HDFS-2815. Namenode sometimes oes not come out of safemode during
+ NN crash + restart. (Uma Maheswara Rao via suresh)
+
Release 0.23.1 - 2012-02-08
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/webhdfs.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/webhdfs.xml?rev=1243691&r1=1243690&r2=1243691&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/webhdfs.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/webhdfs.xml Mon Feb 13 21:00:37 2012
@@ -349,7 +349,7 @@ Hello, webhdfs user!
<ul>
<li>Submit a HTTP PUT request.
<source>
-curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=MKDIRS[&permission=<OCTAL>]"
+ curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=MKDIRS[&permission=<OCTAL>]"
</source>
The client receives a response with a <a href="#boolean"><code>boolean</code> JSON object</a>:
<source>
Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 13 21:00:37 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1159757-1242870
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1159757-1243679
/hadoop/core/branches/branch-0.19/hdfs/src/java:713112
/hadoop/core/branches/branch-0.19/hdfs/src/main/java:713112
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java?rev=1243691&r1=1243690&r2=1243691&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java Mon Feb 13 21:00:37 2012
@@ -133,6 +133,10 @@ class NameNodeConnector {
if (!isBlockTokenEnabled) {
return BlockTokenSecretManager.DUMMY_TOKEN;
} else {
+ if (!shouldRun) {
+ throw new IOException(
+ "Can not get access token. BlockKeyUpdater is not running");
+ }
return blockTokenSecretManager.generateToken(null, eb,
EnumSet.of(BlockTokenSecretManager.AccessMode.REPLACE,
BlockTokenSecretManager.AccessMode.COPY));
@@ -202,16 +206,20 @@ class NameNodeConnector {
*/
class BlockKeyUpdater implements Runnable {
public void run() {
- while (shouldRun) {
- try {
- blockTokenSecretManager.setKeys(namenode.getBlockKeys());
- } catch (Exception e) {
- LOG.error("Failed to set keys", e);
- }
- try {
+ try {
+ while (shouldRun) {
+ try {
+ blockTokenSecretManager.setKeys(namenode.getBlockKeys());
+ } catch (IOException e) {
+ LOG.error("Failed to set keys", e);
+ }
Thread.sleep(keyUpdaterInterval);
- } catch (InterruptedException ie) {
}
+ } catch (InterruptedException e) {
+ LOG.info("InterruptedException in block key updater thread", e);
+ } catch (Throwable e) {
+ LOG.error("Exception in block key updater thread", e);
+ shouldRun = false;
}
}
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java?rev=1243691&r1=1243690&r2=1243691&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java Mon Feb 13 21:00:37 2012
@@ -255,7 +255,6 @@ class BPOfferService {
}
}
-
//This must be called only by blockPoolManager
void start() {
for (BPServiceActor actor : bpServices) {
@@ -666,14 +665,4 @@ class BPOfferService {
return true;
}
- /**
- * Connect to the NN at the given address. This is separated out for ease
- * of testing.
- */
- DatanodeProtocolClientSideTranslatorPB connectToNN(InetSocketAddress nnAddr)
- throws IOException {
- return new DatanodeProtocolClientSideTranslatorPB(nnAddr,
- dn.getConf());
- }
-
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java?rev=1243691&r1=1243690&r2=1243691&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java Mon Feb 13 21:00:37 2012
@@ -189,7 +189,7 @@ class BPServiceActor implements Runnable
private void connectToNNAndHandshake() throws IOException {
// get NN proxy
- bpNamenode = bpos.connectToNN(nnAddr);
+ bpNamenode = dn.connectToNN(nnAddr);
// First phase of the handshake with NN - get the namespace
// info.
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockVolumeChoosingPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockVolumeChoosingPolicy.java?rev=1243691&r1=1243690&r2=1243691&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockVolumeChoosingPolicy.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockVolumeChoosingPolicy.java Mon Feb 13 21:00:37 2012
@@ -28,10 +28,12 @@ import org.apache.hadoop.hdfs.server.dat
* BlockVolumeChoosingPolicy allows a DataNode to
* specify what policy is to be used while choosing
* a volume for a block request.
- *
+ *
+ * Note: This is an evolving i/f and is only for
+ * advanced use.
+ *
***************************************************/
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
+@InterfaceAudience.Private
public interface BlockVolumeChoosingPolicy {
/**
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1243691&r1=1243690&r2=1243691&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Mon Feb 13 21:00:37 2012
@@ -926,6 +926,14 @@ public class DataNode extends Configured
SocketChannel.open().socket() : new Socket();
}
+ /**
+ * Connect to the NN. This is separated out for easier testing.
+ */
+ DatanodeProtocolClientSideTranslatorPB connectToNN(
+ InetSocketAddress nnAddr) throws IOException {
+ return new DatanodeProtocolClientSideTranslatorPB(nnAddr, conf);
+ }
+
public static InterDatanodeProtocol createInterDataNodeProtocolProxy(
DatanodeID datanodeid, final Configuration conf, final int socketTimeout)
throws IOException {
@@ -1893,9 +1901,15 @@ public class DataNode extends Configured
throws IOException {
BPOfferService bpos = blockPoolManager.get(bpid);
if (bpos == null) {
- throw new IOException("cannot find a namnode proxy for bpid=" + bpid);
+ throw new IOException("No block pool offer service for bpid=" + bpid);
}
- return bpos.getActiveNN();
+
+ DatanodeProtocolClientSideTranslatorPB activeNN = bpos.getActiveNN();
+ if (activeNN == null) {
+ throw new IOException(
+ "Block pool " + bpid + " has not recognized an active NN");
+ }
+ return activeNN;
}
/** Block synchronization */
@@ -1904,6 +1918,7 @@ public class DataNode extends Configured
ExtendedBlock block = rBlock.getBlock();
DatanodeProtocolClientSideTranslatorPB nn = getBPNamenode(block
.getBlockPoolId());
+ assert nn != null;
long recoveryId = rBlock.getNewGenerationStamp();
if (LOG.isDebugEnabled()) {
@@ -2236,5 +2251,4 @@ public class DataNode extends Configured
boolean shouldRun() {
return shouldRun;
}
-
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1243691&r1=1243690&r2=1243691&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Mon Feb 13 21:00:37 2012
@@ -2366,7 +2366,6 @@ public class FSNamesystem implements Nam
boolean enforcePermission)
throws AccessControlException, SafeModeException, UnresolvedLinkException,
IOException {
- boolean deleteNow = false;
ArrayList<Block> collectedBlocks = new ArrayList<Block>();
writeLock();
@@ -2385,10 +2384,6 @@ public class FSNamesystem implements Nam
if (!dir.delete(src, collectedBlocks)) {
return false;
}
- deleteNow = collectedBlocks.size() <= BLOCK_DELETION_INCREMENT;
- if (deleteNow) { // Perform small deletes right away
- removeBlocks(collectedBlocks);
- }
} finally {
writeUnlock();
}
@@ -2397,9 +2392,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
- if (!deleteNow) {
- removeBlocks(collectedBlocks); // Incremental deletion of blocks
- }
+ removeBlocks(collectedBlocks); // Incremental deletion of blocks
} finally {
writeUnlock();
}
Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 13 21:00:37 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:1152502-1242870
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:1152502-1243679
/hadoop/core/branches/branch-0.19/hdfs/src/main/native:713112
/hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
/hadoop/core/trunk/src/c++/libhdfs:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 13 21:00:37 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:1159757-1242870
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:1159757-1243679
/hadoop/core/branches/branch-0.19/hdfs/src/main/webapps/datanode:713112
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/datanode:713112
/hadoop/core/trunk/src/webapps/datanode:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 13 21:00:37 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:1152502-1242870
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:1152502-1243679
/hadoop/core/branches/branch-0.19/hdfs/src/main/webapps/hdfs:713112
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/hdfs:713112
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 13 21:00:37 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:1152502-1242870
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:1152502-1243679
/hadoop/core/branches/branch-0.19/hdfs/src/main/webapps/secondary:713112
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/secondary:713112
/hadoop/core/trunk/src/webapps/secondary:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 13 21:00:37 2012
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:1159757-1242870
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:1159757-1243679
/hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs:713112
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs:987665-1095512
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java?rev=1243691&r1=1243690&r2=1243691&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java Mon Feb 13 21:00:37 2012
@@ -59,14 +59,12 @@ public class TestBalancerWithMultipleNam
((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.OFF);
((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.OFF);
((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.OFF);
-// ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.OFF);
}
private static final long CAPACITY = 500L;
private static final String RACK0 = "/rack0";
private static final String RACK1 = "/rack1";
- private static final String RACK2 = "/rack2";
private static final String FILE_NAME = "/tmp.txt";
private static final Path FILE_PATH = new Path(FILE_NAME);
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java?rev=1243691&r1=1243690&r2=1243691&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java Mon Feb 13 21:00:37 2012
@@ -21,7 +21,6 @@ import static org.junit.Assert.*;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.util.Arrays;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -259,29 +258,29 @@ public class TestBPOfferService {
// Have NN1 claim active at txid 1
mockHaStatuses[0] = new NNHAStatusHeartbeat(State.ACTIVE, 1);
- waitForHeartbeats(bpos);
+ bpos.triggerHeartbeatForTests();
assertSame(mockNN1, bpos.getActiveNN());
// NN2 claims active at a higher txid
mockHaStatuses[1] = new NNHAStatusHeartbeat(State.ACTIVE, 2);
- waitForHeartbeats(bpos);
+ bpos.triggerHeartbeatForTests();
assertSame(mockNN2, bpos.getActiveNN());
// Even after another heartbeat from the first NN, it should
// think NN2 is active, since it claimed a higher txid
- waitForHeartbeats(bpos);
+ bpos.triggerHeartbeatForTests();
assertSame(mockNN2, bpos.getActiveNN());
// Even if NN2 goes to standby, DN shouldn't reset to talking to NN1,
// because NN1's txid is lower than the last active txid. Instead,
// it should consider neither active.
mockHaStatuses[1] = new NNHAStatusHeartbeat(State.STANDBY, 2);
- waitForHeartbeats(bpos);
+ bpos.triggerHeartbeatForTests();
assertNull(bpos.getActiveNN());
// Now if NN1 goes back to a higher txid, it should be considered active
mockHaStatuses[0] = new NNHAStatusHeartbeat(State.ACTIVE, 3);
- waitForHeartbeats(bpos);
+ bpos.triggerHeartbeatForTests();
assertSame(mockNN1, bpos.getActiveNN());
} finally {
@@ -302,28 +301,21 @@ public class TestBPOfferService {
/**
* Create a BPOfferService which registers with and heartbeats with the
* specified namenode proxy objects.
+ * @throws IOException
*/
private BPOfferService setupBPOSForNNs(
- DatanodeProtocolClientSideTranslatorPB ... nns) {
+ DatanodeProtocolClientSideTranslatorPB ... nns) throws IOException {
// Set up some fake InetAddresses, then override the connectToNN
// function to return the corresponding proxies.
final Map<InetSocketAddress, DatanodeProtocolClientSideTranslatorPB> nnMap = Maps.newLinkedHashMap();
for (int port = 0; port < nns.length; port++) {
nnMap.put(new InetSocketAddress(port), nns[port]);
+ Mockito.doReturn(nns[port]).when(mockDn).connectToNN(
+ Mockito.eq(new InetSocketAddress(port)));
}
- return new BPOfferService(Lists.newArrayList(nnMap.keySet()), mockDn) {
- @Override
- DatanodeProtocolClientSideTranslatorPB connectToNN(InetSocketAddress nnAddr)
- throws IOException {
- DatanodeProtocolClientSideTranslatorPB nn = nnMap.get(nnAddr);
- if (nn == null) {
- throw new AssertionError("bad NN addr: " + nnAddr);
- }
- return nn;
- }
- };
+ return new BPOfferService(Lists.newArrayList(nnMap.keySet()), mockDn);
}
private void waitForInitialization(final BPOfferService bpos)
@@ -355,30 +347,6 @@ public class TestBPOfferService {
}, 500, 10000);
}
- private void waitForHeartbeats(BPOfferService bpos)
- throws Exception {
- final int countAtStart[];
- synchronized (heartbeatCounts) {
- countAtStart = Arrays.copyOf(
- heartbeatCounts, heartbeatCounts.length);
- }
- bpos.triggerHeartbeatForTests();
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override
- public Boolean get() {
- synchronized (heartbeatCounts) {
- for (int i = 0; i < countAtStart.length; i++) {
- if (heartbeatCounts[i] <= countAtStart[i]) {
- return false;
- }
- }
- return true;
- }
- }
- }, 200, 10000);
- }
-
-
private ReceivedDeletedBlockInfo[] waitForBlockReceived(
ExtendedBlock fakeBlock,
DatanodeProtocolClientSideTranslatorPB mockNN) throws Exception {
Copied: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java (from r1243679, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java?p2=hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java&p1=hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java&r1=1243679&r2=1243691&rev=1243691&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java Mon Feb 13 21:00:37 2012
@@ -42,10 +42,13 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol;
+import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
+import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat.State;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Daemon;
@@ -137,7 +140,9 @@ public class TestBlockRecovery {
Mockito.anyInt(),
Mockito.anyInt(),
Mockito.anyInt()))
- .thenReturn(new DatanodeCommand[0]);
+ .thenReturn(new HeartbeatResponse(
+ new DatanodeCommand[0],
+ new NNHAStatusHeartbeat(State.ACTIVE, 1)));
dn = new DataNode(conf, dirs, null) {
@Override
@@ -147,14 +152,8 @@ public class TestBlockRecovery {
return namenode;
}
};
- dn.runDatanodeDaemon();
- while (!dn.isDatanodeFullyStarted()) {
- try {
- Thread.sleep(50);
- } catch (InterruptedException e) {
- fail("Interrupted starting DN");
- }
- }
+ // Trigger a heartbeat so that it acknowledges the NN as active.
+ dn.getAllBpOs()[0].triggerHeartbeatForTests();
}
/**