You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/01/30 00:28:00 UTC
svn commit: r1562636 - in /manifoldcf/branches/release-1.5-branch: ./
connectors/rss/
connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/
connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/conne...
Author: kwright
Date: Wed Jan 29 23:27:59 2014
New Revision: 1562636
URL: http://svn.apache.org/r1562636
Log:
Pull up fixes for CONNECTORS-872.
Modified:
manifoldcf/branches/release-1.5-branch/ (props changed)
manifoldcf/branches/release-1.5-branch/CHANGES.txt
manifoldcf/branches/release-1.5-branch/connectors/rss/ (props changed)
manifoldcf/branches/release-1.5-branch/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java
manifoldcf/branches/release-1.5-branch/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottleDescription.java
manifoldcf/branches/release-1.5-branch/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java
Propchange: manifoldcf/branches/release-1.5-branch/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk:r1562591,1562634-1562635
Modified: manifoldcf/branches/release-1.5-branch/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/CHANGES.txt?rev=1562636&r1=1562635&r2=1562636&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/CHANGES.txt (original)
+++ manifoldcf/branches/release-1.5-branch/CHANGES.txt Wed Jan 29 23:27:59 2014
@@ -4,6 +4,10 @@ $Id$
======================= Release 1.5 =====================
+CONNECTORS-872: Throttling hang for web connector, as well as web
+connector not obeying rate throttling.
+(Erlend Garåsen, Karl Wright)
+
CONNECTORS-873: SolrCloud now depends on noggit; include this in the
build and dependencies.
(Alessandro Benedetti, Karl Wright)
Propchange: manifoldcf/branches/release-1.5-branch/connectors/rss/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk/connectors/rss:r1562634
Modified: manifoldcf/branches/release-1.5-branch/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java?rev=1562636&r1=1562635&r2=1562636&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java (original)
+++ manifoldcf/branches/release-1.5-branch/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java Wed Jan 29 23:27:59 2014
@@ -193,7 +193,7 @@ public class RSSConnector extends org.ap
{
int maxKBytesPerSecondPerServer = Integer.parseInt(x);
if (maxKBytesPerSecondPerServer > 0)
- minimumMillisecondsPerBytePerServer = ((double)1.0)/(double)maxKBytesPerSecondPerServer;
+ minimumMillisecondsPerBytePerServer = 1000.0/(double)maxKBytesPerSecondPerServer;
}
catch (NumberFormatException e)
{
Modified: manifoldcf/branches/release-1.5-branch/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottleDescription.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottleDescription.java?rev=1562636&r1=1562635&r2=1562636&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottleDescription.java (original)
+++ manifoldcf/branches/release-1.5-branch/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottleDescription.java Wed Jan 29 23:27:59 2014
@@ -91,7 +91,7 @@ public class ThrottleDescription impleme
{
double kbPerSecond = new Double(value).doubleValue();
if (kbPerSecond > 0)
- minMillisecondsPerByte = new Double(((double)1.0)/(double)kbPerSecond);
+ minMillisecondsPerByte = new Double(1000.0/(double)kbPerSecond);
}
catch (NumberFormatException e)
{
@@ -200,6 +200,7 @@ public class ThrottleDescription impleme
}
}
}
+ System.out.println("Minimum milliseconds per byte is "+minMilliseconds);
return minMilliseconds;
}
Modified: manifoldcf/branches/release-1.5-branch/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java?rev=1562636&r1=1562635&r2=1562636&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java (original)
+++ manifoldcf/branches/release-1.5-branch/framework/core/src/main/java/org/apache/manifoldcf/core/throttler/Throttler.java Wed Jan 29 23:27:59 2014
@@ -421,7 +421,7 @@ public class Throttler
* are available in the current pool, across all bins.
*@return the IConnectionThrottler codes for results.
*/
- public int waitConnectionAvailable(String[] binNames, AtomicInteger poolCount)
+ public int waitConnectionAvailable(String[] binNames, AtomicInteger[] poolCounts)
throws InterruptedException
{
// Each bin can signal something different. Bins that signal
@@ -456,7 +456,7 @@ public class Throttler
int result;
try
{
- result = bin.waitConnectionAvailable(poolCount);
+ result = bin.waitConnectionAvailable(poolCounts[i]);
}
catch (Throwable e)
{
@@ -469,7 +469,7 @@ public class Throttler
bin = connectionBins.get(binName);
}
if (bin != null)
- bin.undoReservation(currentRecommendation, poolCount);
+ bin.undoReservation(currentRecommendation, poolCounts[i]);
}
if (e instanceof InterruptedException)
throw (InterruptedException)e;
@@ -491,7 +491,7 @@ public class Throttler
bin = connectionBins.get(binName);
}
if (bin != null)
- bin.undoReservation(currentRecommendation, poolCount);
+ bin.undoReservation(currentRecommendation, poolCounts[i]);
}
return result;
}
@@ -499,7 +499,7 @@ public class Throttler
if (currentRecommendation != IConnectionThrottler.CONNECTION_FROM_NOWHERE && currentRecommendation != result)
{
// Release all previous reservations, including this one, and either return, or retry
- bin.undoReservation(result, poolCount);
+ bin.undoReservation(result, poolCounts[i]);
while (i > 0)
{
i--;
@@ -509,7 +509,7 @@ public class Throttler
bin = connectionBins.get(binName);
}
if (bin != null)
- bin.undoReservation(currentRecommendation, poolCount);
+ bin.undoReservation(currentRecommendation, poolCounts[i]);
}
// Break out of the outer loop so we can retry
@@ -573,7 +573,7 @@ public class Throttler
}
}
- public boolean checkDestroyPooledConnection(String[] binNames, AtomicInteger poolCount)
+ public boolean checkDestroyPooledConnection(String[] binNames, AtomicInteger[] poolCounts)
{
// Only if all believe we can destroy a pool connection, will we do it.
// This is because some pools may be empty, etc.
@@ -588,7 +588,7 @@ public class Throttler
ConnectionBin bin = connectionBins.get(binName);
if (bin != null)
{
- int result = bin.shouldPooledConnectionBeDestroyed(poolCount);
+ int result = bin.shouldPooledConnectionBeDestroyed(poolCounts[i]);
if (result == ConnectionBin.CONNECTION_POOLEMPTY)
{
// Give up now, and undo all the other bins
@@ -597,7 +597,7 @@ public class Throttler
i--;
binName = binNames[i];
bin = connectionBins.get(binName);
- bin.undoPooledConnectionDecision(poolCount);
+ bin.undoPooledConnectionDecision(poolCounts[i]);
}
return false;
}
@@ -613,11 +613,11 @@ public class Throttler
return true;
// Undo pool reservation, since everything is apparently within bounds.
- for (String binName : binNames)
+ for (int j = 0; j < binNames.length; j++)
{
- ConnectionBin bin = connectionBins.get(binName);
+ ConnectionBin bin = connectionBins.get(binNames[j]);
if (bin != null)
- bin.undoPooledConnectionDecision(poolCount);
+ bin.undoPooledConnectionDecision(poolCounts[j]);
}
return false;
@@ -636,7 +636,7 @@ public class Throttler
*@return true if a connection from the pool can be expired. If true is returned, noteConnectionDestruction()
* MUST be called once the connection has actually been destroyed.
*/
- public boolean checkExpireConnection(String[] binNames, AtomicInteger poolCount)
+ public boolean checkExpireConnection(String[] binNames, AtomicInteger[] poolCounts)
{
synchronized (connectionBins)
{
@@ -647,7 +647,7 @@ public class Throttler
ConnectionBin bin = connectionBins.get(binName);
if (bin != null)
{
- if (!bin.hasPooledConnection(poolCount))
+ if (!bin.hasPooledConnection(poolCounts[i]))
{
// Give up now, and undo all the other bins
while (i > 0)
@@ -655,7 +655,7 @@ public class Throttler
i--;
binName = binNames[i];
bin = connectionBins.get(binName);
- bin.undoPooledConnectionDecision(poolCount);
+ bin.undoPooledConnectionDecision(poolCounts[i]);
}
return false;
}
@@ -666,15 +666,15 @@ public class Throttler
}
}
- public void noteConnectionReturnedToPool(String[] binNames, AtomicInteger poolCount)
+ public void noteConnectionReturnedToPool(String[] binNames, AtomicInteger[] poolCounts)
{
synchronized (connectionBins)
{
- for (String binName : binNames)
+ for (int j = 0; j < binNames.length; j++)
{
- ConnectionBin bin = connectionBins.get(binName);
+ ConnectionBin bin = connectionBins.get(binNames[j]);
if (bin != null)
- bin.noteConnectionReturnedToPool(poolCount);
+ bin.noteConnectionReturnedToPool(poolCounts[j]);
}
}
}
@@ -964,16 +964,17 @@ public class Throttler
{
protected final ThrottlingGroup parent;
protected final String[] binNames;
+ protected final AtomicInteger[] poolCounts;
// Keep track of local pool parameters.
- /** This is the number of connections in the pool, times the number of bins per connection */
- protected final AtomicInteger poolCount = new AtomicInteger(0);
-
public ConnectionThrottler(ThrottlingGroup parent, String[] binNames)
{
this.parent = parent;
this.binNames = binNames;
+ this.poolCounts = new AtomicInteger[binNames.length];
+ for (int i = 0; i < poolCounts.length; i++)
+ poolCounts[i] = new AtomicInteger(0);
}
/** Get permission to grab a connection for use. If this object believes there is a connection
@@ -987,7 +988,7 @@ public class Throttler
public int waitConnectionAvailable()
throws InterruptedException
{
- return parent.waitConnectionAvailable(binNames, poolCount);
+ return parent.waitConnectionAvailable(binNames, poolCounts);
}
/** For a new connection, obtain the fetch throttler to use for the connection.
@@ -1023,7 +1024,7 @@ public class Throttler
@Override
public boolean checkDestroyPooledConnection()
{
- return parent.checkDestroyPooledConnection(binNames, poolCount);
+ return parent.checkDestroyPooledConnection(binNames, poolCounts);
}
/** Connection expiration is tricky, because even though a connection may be identified as
@@ -1040,7 +1041,7 @@ public class Throttler
@Override
public boolean checkExpireConnection()
{
- return parent.checkExpireConnection(binNames, poolCount);
+ return parent.checkExpireConnection(binNames, poolCounts);
}
/** Note that a connection has been returned to the pool. Call this method after a connection has been
@@ -1049,7 +1050,7 @@ public class Throttler
@Override
public void noteConnectionReturnedToPool()
{
- parent.noteConnectionReturnedToPool(binNames, poolCount);
+ parent.noteConnectionReturnedToPool(binNames, poolCounts);
}
/** Note that a connection has been destroyed. Call this method ONLY after noteReturnedConnection()