You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sz...@apache.org on 2012/05/11 04:05:37 UTC
svn commit: r1337003 - in
/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apac...
Author: szetszwo
Date: Fri May 11 02:05:31 2012
New Revision: 1337003
URL: http://svn.apache.org/viewvc?rev=1337003&view=rev
Log:
Merge r1335791 through r1337002 from trunk.
Modified:
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/c++/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/block_forensics/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/build-contrib.xml (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/build.xml (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/data_join/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/index/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockPlacementPolicyRaid.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/vaidya/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/examples/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/webapps/job/ (props changed)
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1335791-1337002
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt?rev=1337003&r1=1337002&r2=1337003&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt Fri May 11 02:05:31 2012
@@ -475,6 +475,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4215. RM app page shows 500 error on appid parse error
(Jonathon Eagles via tgraves)
+ MAPREDUCE-4237. TestNodeStatusUpdater can fail if localhost has a domain
+ associated with it (bobby)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1335791-1337002
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1337003&r1=1337002&r2=1337003&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Fri May 11 02:05:31 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.no
import static org.mockito.Mockito.mock;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -35,6 +36,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -88,7 +90,7 @@ public class TestNodeStatusUpdater {
int heartBeatID = 0;
volatile Throwable nmStartError = null;
private final List<NodeId> registeredNodes = new ArrayList<NodeId>();
- private final Configuration conf = new YarnConfiguration();
+ private final Configuration conf = createNMConfig();
private NodeManager nm;
protected NodeManager rebootedNodeManager;
@@ -117,7 +119,9 @@ public class TestNodeStatusUpdater {
Resource resource = request.getResource();
LOG.info("Registering " + nodeId.toString());
// NOTE: this really should be checking against the config value
- Assert.assertEquals("localhost:12345", nodeId.toString());
+ InetSocketAddress expected = NetUtils.getConnectAddress(
+ conf.getSocketAddr(YarnConfiguration.NM_ADDRESS, null, -1));
+ Assert.assertEquals(NetUtils.getHostPortString(expected), nodeId.toString());
Assert.assertEquals(5 * 1024, resource.getMemory());
registeredNodes.add(nodeId);
RegistrationResponse regResponse = recordFactory
@@ -429,6 +433,7 @@ public class TestNodeStatusUpdater {
while (nm.getServiceState() == STATE.INITED && waitCount++ != 20) {
LOG.info("Waiting for NM to start..");
if (nmStartError != null) {
+ LOG.error("Error during startup. ", nmStartError);
Assert.fail(nmStartError.getCause().getMessage());
}
Thread.sleep(1000);
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/c++/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/c++:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/block_forensics/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/build.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/data_join/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:r1335791-1337002
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java?rev=1337003&r1=1337002&r2=1337003&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java Fri May 11 02:05:31 2012
@@ -144,25 +144,25 @@ public class BlockPlacementPolicyRaid ex
/** {@inheritDoc} */
@Override
- public DatanodeDescriptor chooseReplicaToDelete(BlockCollection inode,
+ public DatanodeDescriptor chooseReplicaToDelete(BlockCollection bc,
Block block, short replicationFactor,
Collection<DatanodeDescriptor> first,
Collection<DatanodeDescriptor> second) {
DatanodeDescriptor chosenNode = null;
try {
- String path = cachedFullPathNames.get(inode);
+ String path = cachedFullPathNames.get(bc);
FileType type = getFileType(path);
if (type == FileType.NOT_RAID) {
return defaultPolicy.chooseReplicaToDelete(
- inode, block, replicationFactor, first, second);
+ bc, block, replicationFactor, first, second);
}
List<LocatedBlock> companionBlocks =
getCompanionBlocks(path, type, block);
if (companionBlocks == null || companionBlocks.size() == 0) {
// Use the default method if it is not a valid raided or parity file
return defaultPolicy.chooseReplicaToDelete(
- inode, block, replicationFactor, first, second);
+ bc, block, replicationFactor, first, second);
}
// Delete from the first collection first
// This ensures the number of unique rack of this block is not reduced
@@ -174,12 +174,12 @@ public class BlockPlacementPolicyRaid ex
return chosenNode;
}
return defaultPolicy.chooseReplicaToDelete(
- inode, block, replicationFactor, first, second);
+ bc, block, replicationFactor, first, second);
} catch (Exception e) {
LOG.debug("Error happend when choosing replica to delete" +
StringUtils.stringifyException(e));
return defaultPolicy.chooseReplicaToDelete(
- inode, block, replicationFactor, first, second);
+ bc, block, replicationFactor, first, second);
}
}
@@ -446,25 +446,25 @@ public class BlockPlacementPolicyRaid ex
};
static private class INodeWithHashCode {
- BlockCollection inode;
- INodeWithHashCode(BlockCollection inode) {
- this.inode = inode;
+ BlockCollection bc;
+ INodeWithHashCode(BlockCollection bc) {
+ this.bc= bc;
}
@Override
public boolean equals(Object obj) {
- return inode == obj;
+ return bc== obj;
}
@Override
public int hashCode() {
- return System.identityHashCode(inode);
+ return System.identityHashCode(bc);
}
String getFullPathName() {
- return inode.getName();
+ return bc.getName();
}
}
- public String get(BlockCollection inode) throws IOException {
- return cacheInternal.get(new INodeWithHashCode(inode));
+ public String get(BlockCollection bc) throws IOException {
+ return cacheInternal.get(new INodeWithHashCode(bc));
}
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockPlacementPolicyRaid.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockPlacementPolicyRaid.java?rev=1337003&r1=1337002&r2=1337003&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockPlacementPolicyRaid.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockPlacementPolicyRaid.java Fri May 11 02:05:31 2012
@@ -240,19 +240,19 @@ public class TestBlockPlacementPolicyRai
// test full path cache
CachedFullPathNames cachedFullPathNames =
new CachedFullPathNames(namesystem);
- final BlockCollection[] inodes = NameNodeRaidTestUtil.getBlockCollections(
+ final BlockCollection[] bcs = NameNodeRaidTestUtil.getBlockCollections(
namesystem, file1, file2);
- verifyCachedFullPathNameResult(cachedFullPathNames, inodes[0]);
- verifyCachedFullPathNameResult(cachedFullPathNames, inodes[0]);
- verifyCachedFullPathNameResult(cachedFullPathNames, inodes[1]);
- verifyCachedFullPathNameResult(cachedFullPathNames, inodes[1]);
+ verifyCachedFullPathNameResult(cachedFullPathNames, bcs[0]);
+ verifyCachedFullPathNameResult(cachedFullPathNames, bcs[0]);
+ verifyCachedFullPathNameResult(cachedFullPathNames, bcs[1]);
+ verifyCachedFullPathNameResult(cachedFullPathNames, bcs[1]);
try {
Thread.sleep(1200L);
} catch (InterruptedException e) {
}
- verifyCachedFullPathNameResult(cachedFullPathNames, inodes[1]);
- verifyCachedFullPathNameResult(cachedFullPathNames, inodes[0]);
+ verifyCachedFullPathNameResult(cachedFullPathNames, bcs[1]);
+ verifyCachedFullPathNameResult(cachedFullPathNames, bcs[0]);
} finally {
if (cluster != null) {
cluster.shutdown();
@@ -476,14 +476,14 @@ public class TestBlockPlacementPolicyRai
}
private void verifyCachedFullPathNameResult(
- CachedFullPathNames cachedFullPathNames, BlockCollection inode)
+ CachedFullPathNames cachedFullPathNames, BlockCollection bc)
throws IOException {
- String res1 = inode.getName();
- String res2 = cachedFullPathNames.get(inode);
+ String res1 = bc.getName();
+ String res2 = cachedFullPathNames.get(bc);
LOG.info("Actual path name: " + res1);
LOG.info("Cached path name: " + res2);
- Assert.assertEquals(cachedFullPathNames.get(inode),
- inode.getName());
+ Assert.assertEquals(cachedFullPathNames.get(bc),
+ bc.getName());
}
private void verifyCachedBlocksResult(CachedLocatedBlocks cachedBlocks,
@@ -502,7 +502,7 @@ public class TestBlockPlacementPolicyRai
private Collection<LocatedBlock> getCompanionBlocks(
FSNamesystem namesystem, BlockPlacementPolicyRaid policy,
ExtendedBlock block) throws IOException {
- INodeFile inode = (INodeFile)blockManager.blocksMap.getINode(block
+ INodeFile inode = (INodeFile)blockManager.blocksMap.getBlockCollection(block
.getLocalBlock());
FileType type = policy.getFileType(inode.getFullPathName());
return policy.getCompanionBlocks(inode.getFullPathName(), type,
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/examples:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/java:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:r1335791-1337002
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:r1335791-1337002