svn commit: r227055 - /lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java

[mc...@apache.org]

Author: mc
Date: Tue Aug  2 11:14:30 2005
New Revision: 227055

URL: http://svn.apache.org/viewcvs?rev=227055&view=rev
Log:

  We can have multiple datanodes on the same machine.  When
replicating, we want to make sure that replicates don't get
assigned to the same physical machine.

  (We might want to relax this requirement in the future.  E.g.,
if there are 3 replicates, maybe 2 of them can be on the same
machine.)


Modified:
    lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java

Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java?rev=227055&r1=227054&r2=227055&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java Tue Aug  2 11:14:30 2005
@@ -1143,47 +1143,57 @@
      * choose according to capacity and load-balancing needs (or even 
      * network-topology, to avoid inter-switch traffic).
      */
-    DatanodeInfo chooseTarget(TreeSet alreadyHasNode, TreeSet alreadyChosen) {
+    DatanodeInfo chooseTarget(TreeSet forbidden1, TreeSet forbidden2) {
+        //
+        // Check if there are any available targets at all
+        //
         int totalMachines = datanodeMap.size();
         if (totalMachines == 0) {
             LOG.info("While choosing target, totalMachines is " + totalMachines);
             return null;
         }
-        int freeMachines = totalMachines;
+
+        //
+        // In addition to already-chosen datanode/port pairs, we want to avoid
+        // already-chosen machinenames.  (There can be multiple datanodes per
+        // machine.)  We might relax this requirement in the future, though. (Maybe
+        // so that at least one replicate is off the machine.)
+        //
+        TreeSet forbiddenMachines = new TreeSet();
+        if (forbidden1 != null) {
+            for (Iterator it = forbidden1.iterator(); it.hasNext(); ) {
+                DatanodeInfo cur = (DatanodeInfo) it.next();
+                forbiddenMachines.add(cur.getHost());
+            }
+        }
+        if (forbidden2 != null) {
+            for (Iterator it = forbidden2.iterator(); it.hasNext(); ) {
+                DatanodeInfo cur = (DatanodeInfo) it.next();
+                forbiddenMachines.add(cur.getHost());
+            }
+        }
+
+        //
+        // Now build list of machines we can actually choose from
+        //
+        Vector targetList = new Vector();
         for (Iterator it = datanodeMap.values().iterator(); it.hasNext(); ) {
             DatanodeInfo node = (DatanodeInfo) it.next();
-            if ((alreadyHasNode != null && alreadyHasNode.contains(node)) ||
-                (alreadyChosen != null && alreadyChosen.contains(node))) {
-                freeMachines--;
+            if ((forbidden1 == null || ! forbidden1.contains(node)) &&
+                (forbidden2 == null || ! forbidden2.contains(node)) &&
+                (! forbiddenMachines.contains(node.getHost()))) {
+                targetList.add(node);
             }
         }
 
         //
         // Now pick one
         //
-        DatanodeInfo target = null;
-        if (freeMachines > 0) {
-            //
-            // Get all possible targets
-            //
-            int i = 0;
-            DatanodeInfo targetlist[] = new DatanodeInfo[totalMachines];
-            for (Iterator it = datanodeMap.values().iterator(); it.hasNext(); i++) {
-                targetlist[i] = (DatanodeInfo) it.next();
-            }
-        
-            do {
-                int index = r.nextInt(totalMachines);
-                target = targetlist[index];
-
-                if ((alreadyHasNode != null && alreadyHasNode.contains(target)) ||
-                    (alreadyChosen != null && alreadyChosen.contains(target))) {
-                    target = null;
-                }
-            } while (target == null);
+        if (targetList.size() == 0) {
+            return null;
+        } else {
+            return (DatanodeInfo) targetList.elementAt(r.nextInt() % targetList.size());
         }
-        return target;
-
         /**
          * Choose target weighted by available storage
          */