svn commit: r525593 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/FSNamesystem.java

[to...@apache.org]

Author: tomwhite
Date: Wed Apr  4 12:46:26 2007
New Revision: 525593

URL: http://svn.apache.org/viewvc?view=rev&rev=525593
Log:
HADOOP-1187.  Improve DFS Scalability: avoid scanning entire list of datanodes in getAdditionalBlocks.  Contributed by Dhruba Borthakur.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=525593&r1=525592&r2=525593
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed Apr  4 12:46:26 2007
@@ -103,6 +103,9 @@
 32. HADOOP-1194.  Make compression style record level for map output
     compression.  (Arun C Murthy via tomwhite)
 
+33. HADOOP-1187.  Improve DFS Scalability: avoid scanning entire list of
+    datanodes in getAdditionalBlocks.  (Dhruba Borthakur via tomwhite)
+
 
 Release 0.12.3 (not yet released)
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?view=diff&rev=525593&r1=525592&r2=525593
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Wed Apr  4 12:46:26 2007
@@ -561,6 +561,7 @@
         if (blocks != null) {
             results = new Object[2];
             DatanodeDescriptor machineSets[][] = new DatanodeDescriptor[blocks.length][];
+            DatanodeDescriptor clientNode = getDatanodeByHost(clientMachine);
 
             for (int i = 0; i < blocks.length; i++) {
                 Collection<DatanodeDescriptor> containingNodes = blocksMap.get(blocks[i]);
@@ -573,7 +574,7 @@
                     containingNodesList.addAll(containingNodes);
                     
                     machineSets[i] = replicator.sortByDistance(
-                        getDatanodeByHost(clientMachine), containingNodesList);
+                        clientNode, containingNodesList);
                 }
             }
 
@@ -745,8 +746,9 @@
         }
 
         // Get the array of replication targets
+        DatanodeDescriptor clientNode = getDatanodeByHost(clientMachine.toString());
         DatanodeDescriptor targets[] = replicator.chooseTarget(replication,
-            getDatanodeByHost(clientMachine.toString()), null, blockSize);
+                                                      clientNode, null, blockSize);
         if (targets.length < this.minReplication) {
             if (clusterMap.getNumOfLeaves() == 0) {
               throw new IOException("Failed to create file "+src
@@ -768,7 +770,8 @@
                            new FileUnderConstruction(replication, 
                                                      blockSize,
                                                      holder,
-                                                     clientMachine));
+                                                     clientMachine, 
+                                                     clientNode));
         NameNode.stateChangeLog.debug( "DIR* NameSystem.startFile: "
                    +"add "+src+" to pendingCreates for "+holder );
         synchronized (leases) {
@@ -837,10 +840,10 @@
         }
         
         // Get the array of replication targets
-        String clientHost = pendingFile.getClientMachine().toString();
+        DatanodeDescriptor clientNode = pendingFile.getClientNode();
         DatanodeDescriptor targets[] = replicator.chooseTarget(
             (int)(pendingFile.getReplication()),
-            getDatanodeByHost(clientHost),
+            clientNode,
             null,
             pendingFile.getBlockSize());
         if (targets.length < this.minReplication) {
@@ -3355,16 +3358,19 @@
       private Collection<Block> blocks;
       private UTF8 clientName;         // lease holder
       private UTF8 clientMachine;
+      private DatanodeDescriptor clientNode; // if client is a cluster node too.
       
       FileUnderConstruction(short replication,
                             long blockSize,
                             UTF8 clientName,
-                            UTF8 clientMachine) throws IOException {
+                            UTF8 clientMachine,
+                            DatanodeDescriptor clientNode) throws IOException {
         this.blockReplication = replication;
         this.blockSize = blockSize;
         this.blocks = new ArrayList<Block>();
         this.clientName = clientName;
         this.clientMachine = clientMachine;
+        this.clientNode = clientNode;
       }
       
       public short getReplication() {
@@ -3385,6 +3391,10 @@
       
       public UTF8 getClientMachine() {
         return clientMachine;
+      }
+
+      public DatanodeDescriptor getClientNode() {
+        return clientNode;
       }
     }