svn commit: r1551737 - in /manifoldcf/trunk: connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ framework/core/src/main/java/org/apache/manifoldcf/core/throttler/

[kw...@apache.org]

Author: kwright
Date: Tue Dec 17 22:24:40 2013
New Revision: 1551737

URL: http://svn.apache.org/r1551737
Log:
Fix infinite loop issue with closing idle connections.  Part of CONNECTORS-829.

Modified:
    manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
    manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/ConnectionBin.java
    manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java

Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java?rev=1551737&r1=1551736&r2=1551737&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java (original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java Tue Dec 17 22:24:40 2013
@@ -2009,6 +2009,8 @@ public class ThrottledFetcher
           connection.destroy();
           connectionThrottler.noteConnectionDestroyed();
         }
+        else
+          break;
       }
     }
     

Modified: manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/ConnectionBin.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/ConnectionBin.java?rev=1551737&r1=1551736&r2=1551737&view=diff
==============================================================================
--- manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/ConnectionBin.java (original)
+++ manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/ConnectionBin.java Tue Dec 17 22:24:40 2013
@@ -198,6 +198,19 @@ public class ConnectionBin
     return CONNECTION_POOLEMPTY;
   }
 
+  /** Check only if there's a pooled connection, and make moves to take it from the pool.
+  */
+  public synchronized boolean hasPooledConnection(AtomicInteger poolCount)
+  {
+    int currentPoolCount = poolCount.get();
+    if (currentPoolCount > 0)
+    {
+      poolCount.set(currentPoolCount-1);
+      return true;
+    }
+    return false;
+  }
+  
   /** Undo the decision to destroy a pooled connection.
   */
   public synchronized void undoPooledConnectionDecision(AtomicInteger poolCount)

Modified: manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java?rev=1551737&r1=1551736&r2=1551737&view=diff
==============================================================================
--- manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java (original)
+++ manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java Tue Dec 17 22:24:40 2013
@@ -583,6 +583,47 @@ public class Throttler
 
     }
     
+    /** Connection expiration is tricky, because even though a connection may be identified as
+    * being expired, at the very same moment it could be handed out in another thread.  So there
+    * is a natural race condition present.
+    * The way the connection throttler deals with that is to allow the caller to reserve a connection
+    * for expiration.  This must be called BEFORE the actual identified connection is removed from the
+    * connection pool.  If the value returned by this method is "true", then a connection MUST be removed
+    * from the pool and destroyed, whether or not the identified connection is actually still available for
+    * destruction or not.
+    *@return true if a connection from the pool can be expired.  If true is returned, noteConnectionDestruction()
+    *  MUST be called once the connection has actually been destroyed.
+    */
+    public boolean checkExpireConnection(String[] binNames, AtomicInteger poolCount)
+    {
+      synchronized (connectionBins)
+      {
+        int i = 0;
+        while (i < binNames.length)
+        {
+          String binName = binNames[i];
+          ConnectionBin bin = connectionBins.get(binName);
+          if (bin != null)
+          {
+            if (!bin.hasPooledConnection(poolCount))
+            {
+              // Give up now, and undo all the other bins
+              while (i > 0)
+              {
+                i--;
+                binName = binNames[i];
+                bin = connectionBins.get(binName);
+                bin.undoPooledConnectionDecision(poolCount);
+              }
+              return false;
+            }
+          }
+          i++;
+        }
+        return true;
+      }
+    }
+
     public void noteConnectionReturnedToPool(String[] binNames, AtomicInteger poolCount)
     {
       synchronized (connectionBins)
@@ -957,8 +998,7 @@ public class Throttler
     @Override
     public boolean checkExpireConnection()
     {
-      // MHL
-      return false;
+      return parent.checkExpireConnection(binNames, poolCount);
     }
     
     /** Note that a connection has been returned to the pool.  Call this method after a connection has been