You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/04/26 21:36:04 UTC
svn commit: r1476354 - in /manifoldcf/trunk/connectors:
rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/
webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/
Author: kwright
Date: Fri Apr 26 19:36:03 2013
New Revision: 1476354
URL: http://svn.apache.org/r1476354
Log:
Another fix for throttling (CONNECTORS-679). Use one synchronizer instead of two. This may make some things slightly slower, but I believe our current use allowed unsynchronized variables to be modified.
Modified:
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
Modified: manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java?rev=1476354&r1=1476353&r2=1476354&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java (original)
+++ manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java Fri Apr 26 19:36:03 2013
@@ -978,9 +978,6 @@ public class ThrottledFetcher
/** Total actual bytes read in this series; this includes fetches in progress */
protected long totalBytesRead = -1L;
- /** This object is used to gate access while the first chunk is being read */
- protected Integer firstChunkLock = new Integer(0);
-
/** Outstanding connection counter */
protected volatile int outstandingConnections = 0;
@@ -1102,19 +1099,16 @@ public class ThrottledFetcher
long currentTime = System.currentTimeMillis();
- synchronized (firstChunkLock)
+ synchronized (this)
{
while (estimateInProgress)
- firstChunkLock.wait();
+ wait();
if (estimateValid == false)
{
seriesStartTime = currentTime;
estimateInProgress = true;
// Add these bytes to the estimated total
- synchronized (this)
- {
- totalBytesRead += (long)byteCount;
- }
+ totalBytesRead += (long)byteCount;
// Exit early; this thread isn't going to do any waiting
//if (Logging.connectors.isTraceEnabled())
// Logging.connectors.trace("RSS: Read begin noted; gathering stats for '"+serverName+"'");
@@ -1161,11 +1155,21 @@ public class ThrottledFetcher
{
if (!finished)
{
- if (estimateInProgress)
- {
- estimateInProgress = false;
- firstChunkLock.notifyAll();
- }
+ abortRead();
+ }
+ }
+ }
+
+ /** Abort a read in progress.
+ */
+ public void abortRead()
+ {
+ synchronized (this)
+ {
+ if (estimateInProgress)
+ {
+ estimateInProgress = false;
+ notifyAll();
}
}
}
@@ -1183,11 +1187,6 @@ public class ThrottledFetcher
synchronized (this)
{
totalBytesRead = totalBytesRead + (long)actualCount - (long)originalCount;
- }
-
- // Only one thread should get here if it's the first chunk, but we synchronize to be sure
- synchronized (firstChunkLock)
- {
if (estimateInProgress)
{
if (actualCount == 0)
@@ -1197,7 +1196,7 @@ public class ThrottledFetcher
rateEstimate = ((double)(currentTime - seriesStartTime))/(double)actualCount;
estimateValid = true;
estimateInProgress = false;
- firstChunkLock.notifyAll();
+ notifyAll();
}
}
Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java?rev=1476354&r1=1476353&r2=1476354&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java (original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java Fri Apr 26 19:36:03 2013
@@ -769,9 +769,6 @@ public class ThrottledFetcher
/** Total actual bytes read in this series; this includes fetches in progress */
protected long totalBytesRead = -1L;
- /** This object is used to gate access while the first chunk is being read */
- protected Integer firstChunkLock = new Integer(0);
-
/** Constructor. */
public ThrottleBin(String binName)
{
@@ -814,19 +811,16 @@ public class ThrottledFetcher
{
long currentTime = System.currentTimeMillis();
- synchronized (firstChunkLock)
+ synchronized (this)
{
while (estimateInProgress)
- firstChunkLock.wait();
+ wait();
if (estimateValid == false)
{
seriesStartTime = currentTime;
estimateInProgress = true;
// Add these bytes to the estimated total
- synchronized (this)
- {
- totalBytesRead += (long)byteCount;
- }
+ totalBytesRead += (long)byteCount;
// Exit early; this thread isn't going to do any waiting
return;
}
@@ -867,15 +861,25 @@ public class ThrottledFetcher
{
if (!finished)
{
- if (estimateInProgress)
- {
- estimateInProgress = false;
- firstChunkLock.notifyAll();
- }
+ abortRead();
}
}
}
+ /** Abort a read in progress.
+ */
+ public void abortRead()
+ {
+ synchronized (this)
+ {
+ if (estimateInProgress)
+ {
+ estimateInProgress = false;
+ notifyAll();
+ }
+ }
+ }
+
/** Note the end of an individual read from the server. Call this just after an individual read completes.
* Pass the actual number of bytes read to the method.
*/
@@ -886,11 +890,6 @@ public class ThrottledFetcher
synchronized (this)
{
totalBytesRead = totalBytesRead + (long)actualCount - (long)originalCount;
- }
-
- // Only one thread should get here if it's the first chunk, but we synchronize to be sure
- synchronized (firstChunkLock)
- {
if (estimateInProgress)
{
if (actualCount == 0)
@@ -900,7 +899,7 @@ public class ThrottledFetcher
rateEstimate = ((double)(currentTime - seriesStartTime))/(double)actualCount;
estimateValid = true;
estimateInProgress = false;
- firstChunkLock.notifyAll();
+ notifyAll();
}
}
}
@@ -1179,11 +1178,24 @@ public class ThrottledFetcher
throws InterruptedException
{
// Consult with throttle bins
- int i = 0;
- while (i < throttleBinArray.length)
+ int lastOneDone = 0;
+ try
{
- throttleBinArray[i].beginRead(len,minMillisecondsPerByte[i]);
- i++;
+ for (int i = 0; i < throttleBinArray.length; i++)
+ {
+ throttleBinArray[i].beginRead(len,minMillisecondsPerByte[i]);
+ lastOneDone = i + 1;
+ }
+ }
+ finally
+ {
+ if (lastOneDone != throttleBinArray.length)
+ {
+ for (int i = 0; i < lastOneDone; i++)
+ {
+ throttleBinArray[i].abortRead();
+ }
+ }
}
}
@@ -1191,11 +1203,9 @@ public class ThrottledFetcher
public void endRead(int origLen, int actualAmt)
{
// Consult with throttle bins
- int i = 0;
- while (i < throttleBinArray.length)
+ for (int i = 0; i < throttleBinArray.length; i++)
{
throttleBinArray[i].endRead(origLen,actualAmt);
- i++;
}
}