You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dd...@apache.org on 2009/03/17 13:37:40 UTC
svn commit: r755222 - in /hadoop/core/branches/branch-0.20: ./ CHANGES.txt
src/core/org/apache/hadoop/net/NetworkTopology.java
src/mapred/org/apache/hadoop/mapred/FileInputFormat.java
src/test/org/apache/hadoop/mapred/TestGetSplitHosts.java
Author: ddas
Date: Tue Mar 17 12:37:39 2009
New Revision: 755222
URL: http://svn.apache.org/viewvc?rev=755222&view=rev
Log:
HADOOP-5103. Committing the patch to 0.20 branch.
Modified:
hadoop/core/branches/branch-0.20/ (props changed)
hadoop/core/branches/branch-0.20/CHANGES.txt (contents, props changed)
hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/net/NetworkTopology.java
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestGetSplitHosts.java
Propchange: hadoop/core/branches/branch-0.20/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 17 12:37:39 2009
@@ -1,2 +1,2 @@
/hadoop/core/branches/branch-0.19:713112
-/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035
+/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035
Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=755222&r1=755221&r2=755222&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Tue Mar 17 12:37:39 2009
@@ -735,6 +735,10 @@
making sure that lease renewal thread exits before dfs client exits.
(hairong)
+ HADOOP-5103. FileInputFormat now reuses the clusterMap network topology object
+ and that brings down the log messages in the JobClient to do with
+ NetworkTopology.add significantly. (Jothi Padmanabhan via ddas)
+
Release 0.19.2 - Unreleased
BUG FIXES
Propchange: hadoop/core/branches/branch-0.20/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 17 12:37:39 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
-/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035
+/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035
Modified: hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/net/NetworkTopology.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/net/NetworkTopology.java?rev=755222&r1=755221&r2=755222&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/net/NetworkTopology.java (original)
+++ hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/net/NetworkTopology.java Tue Mar 17 12:37:39 2009
@@ -316,7 +316,6 @@
throw new IllegalArgumentException(
"Not allow to add an inner node: "+NodeBase.getPath(node));
}
- LOG.info("Adding a new node: "+NodeBase.getPath(node));
netlock.writeLock().lock();
try {
Node rack = getNode(node.getNetworkLocation());
@@ -326,6 +325,7 @@
+ " at an illegal network location");
}
if (clusterMap.add(node)) {
+ LOG.info("Adding a new node: "+NodeBase.getPath(node));
if (rack == null) {
numOfRacks++;
}
Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java?rev=755222&r1=755221&r2=755222&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java (original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java Tue Mar 17 12:37:39 2009
@@ -214,6 +214,7 @@
// generate splits
ArrayList<FileSplit> splits = new ArrayList<FileSplit>(numSplits);
+ NetworkTopology clusterMap = new NetworkTopology();
for (FileStatus file: files) {
Path path = file.getPath();
FileSystem fs = path.getFileSystem(job);
@@ -226,7 +227,7 @@
long bytesRemaining = length;
while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {
String[] splitHosts = getSplitHosts(blkLocations,
- length-bytesRemaining, splitSize);
+ length-bytesRemaining, splitSize, clusterMap);
splits.add(new FileSplit(path, length-bytesRemaining, splitSize,
splitHosts));
bytesRemaining -= splitSize;
@@ -237,7 +238,7 @@
blkLocations[blkLocations.length-1].getHosts()));
}
} else if (length != 0) {
- String[] splitHosts = getSplitHosts(blkLocations,0,length);
+ String[] splitHosts = getSplitHosts(blkLocations,0,length,clusterMap);
splits.add(new FileSplit(path, 0, length, splitHosts));
} else {
//Create empty hosts array for zero length files
@@ -417,7 +418,8 @@
* @throws IOException
*/
protected String[] getSplitHosts(BlockLocation[] blkLocations,
- long offset, long splitSize) throws IOException {
+ long offset, long splitSize, NetworkTopology clusterMap)
+ throws IOException {
int startIndex = getBlockIndex(blkLocations, offset);
@@ -442,7 +444,6 @@
long bytesInLastBlock = bytesInThisBlock;
int endIndex = index - 1;
- NetworkTopology clusterMap = new NetworkTopology();
Map <Node,NodeInfo> hostsMap = new IdentityHashMap<Node,NodeInfo>();
Map <Node,NodeInfo> racksMap = new IdentityHashMap<Node,NodeInfo>();
String [] allTopos = new String[0];
@@ -486,6 +487,11 @@
if (node == null) {
node = new NodeBase(topo);
clusterMap.add(node);
+ }
+
+ nodeInfo = hostsMap.get(node);
+
+ if (nodeInfo == null) {
nodeInfo = new NodeInfo(node);
hostsMap.put(node,nodeInfo);
parentNode = node.getParent();
Modified: hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestGetSplitHosts.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestGetSplitHosts.java?rev=755222&r1=755221&r2=755222&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestGetSplitHosts.java (original)
+++ hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestGetSplitHosts.java Tue Mar 17 12:37:39 2009
@@ -18,6 +18,8 @@
package org.apache.hadoop.mapred;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.net.NetworkTopology;
+
import junit.framework.TestCase;
public class TestGetSplitHosts extends TestCase {
@@ -28,6 +30,7 @@
int block1Size = 100, block2Size = 150, block3Size = 75;
int fileSize = block1Size + block2Size + block3Size;
int replicationFactor = 3;
+ NetworkTopology clusterMap = new NetworkTopology();
BlockLocation[] bs = new BlockLocation[numBlocks];
@@ -72,7 +75,7 @@
SequenceFileInputFormat< String, String> sif =
new SequenceFileInputFormat<String,String>();
- String [] hosts = sif.getSplitHosts(bs, 0, fileSize);
+ String [] hosts = sif.getSplitHosts(bs, 0, fileSize, clusterMap);
// Contributions By Racks are
// Rack1 175
@@ -93,7 +96,7 @@
bs[2] = new BlockLocation(block3Names,block3Hosts,block1Size+block2Size,
block3Size);
- hosts = sif.getSplitHosts(bs, 0, fileSize);
+ hosts = sif.getSplitHosts(bs, 0, fileSize, clusterMap);
// host1 makes the highest contribution among all hosts
// So, that should be returned before others