You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/12/09 16:09:02 UTC
svn commit: r1549594 - in /manifoldcf/trunk: ./
framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/
framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/
framework/pull-agent/src/main/java/or...
Author: kwright
Date: Mon Dec 9 15:09:02 2013
New Revision: 1549594
URL: http://svn.apache.org/r1549594
Log:
Fix for CONNECTORS-831.
Added:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java
- copied, changed from r1549500, manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/ReprioritizationTrackerFactory.java (with props)
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java
- copied, changed from r1549500, manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java
Removed:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupThread.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireThread.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PriorityCalculator.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingActivity.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1549594&r1=1549593&r2=1549594&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Mon Dec 9 15:09:02 2013
@@ -3,6 +3,10 @@ $Id$
======================= 1.5-dev =====================
+CONNECTORS-831: Refactor ReprioritizationTracker so that it uses
+the interface/factory paradigm.
+(Karl Wright)
+
CONNECTORS-830: Stuffer thread throttling now works globally, not
just locally.
(Karl Wright)
Copied: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java (from r1549500, manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java)
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java?p2=manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java&p1=manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java&r1=1549500&r2=1549594&rev=1549594&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java Mon Dec 9 15:09:02 2013
@@ -16,47 +16,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.manifoldcf.crawler.system;
+package org.apache.manifoldcf.crawler.interfaces;
import org.apache.manifoldcf.core.interfaces.*;
-import org.apache.manifoldcf.agents.interfaces.*;
-import org.apache.manifoldcf.crawler.interfaces.*;
-import java.util.*;
-import java.io.*;
-/** This class tracks cluster-wide reprioritization operations.
+/** This interface represents functionality that tracks cluster-wide
+* reprioritization operations.
* These operations are driven forward by whatever thread needs them,
* and are completed if those processes die by the threads that clean up
* after the original process.
*/
-public class ReprioritizationTracker
+public interface IReprioritizationTracker
{
public static final String _rcsid = "@(#)$Id$";
- protected final static String trackerWriteLock = "_REPR_TRACKER_LOCK_";
- protected final static String trackerProcessIDResource = "_REPR_TRACKER_PID_";
- protected final static String trackerReproIDResource = "_REPR_TRACKER_RID_";
- protected final static String trackerTimestampResource = "_REPR_TIMESTAMP_";
- protected final static String trackerMinimumDepthResource = "_REPR_MINDEPTH_";
-
- /** Lock manager */
- protected final ILockManager lockManager;
- protected final IBinManager binManager;
-
- /** Preload requests */
- protected final Map<String,PreloadRequest> preloadRequests = new HashMap<String,PreloadRequest>();
- /** Preload values */
- protected final Map<String,PreloadedValues> preloadedValues = new HashMap<String,PreloadedValues>();
-
- /** Constructor.
- */
- public ReprioritizationTracker(IThreadContext threadContext)
- throws ManifoldCFException
- {
- lockManager = LockManagerFactory.make(threadContext);
- binManager = BinManagerFactory.make(threadContext);
- }
-
/** Start a reprioritization activity.
*@param prioritizationTime is the timestamp of the prioritization.
*@param processID is the process ID of the process performing/waiting for the prioritization
@@ -64,121 +37,28 @@ public class ReprioritizationTracker
*@param reproID is the reprocessing thread ID
*/
public void startReprioritization(long prioritizationTime, String processID, String reproID)
- throws ManifoldCFException
- {
- lockManager.enterWriteLock(trackerWriteLock);
- try
- {
- Long currentTime = readTime();
- String currentProcessID = readProcessID();
- if (currentTime != null && currentProcessID != null)
- {
- // Already a reprioritization in progress.
- if (prioritizationTime <= currentTime.longValue())
- return;
- }
- writeTime(new Long(prioritizationTime));
- writeProcessID(processID);
- writeReproID(reproID);
- try
- {
- binManager.reset();
- }
- catch (Throwable e)
- {
- writeTime(null);
- writeProcessID(null);
- writeReproID(null);
- if (e instanceof Error)
- throw (Error)e;
- else if (e instanceof RuntimeException)
- throw (RuntimeException)e;
- else if (e instanceof ManifoldCFException)
- throw (ManifoldCFException)e;
- else
- throw new RuntimeException("Unknown exception: "+e.getClass().getName()+": "+e.getMessage(),e);
- }
- writeMinimumDepth(0.0);
- }
- finally
- {
- lockManager.leaveWriteLock(trackerWriteLock);
- }
- }
-
+ throws ManifoldCFException;
/** Retrieve the current reprioritization time stamp. This should be obtained before
* performing any prioritization steps.
*@return the current prioritization timestamp, or null if no prioritization is in effect.
*/
public Long checkReprioritizationInProgress()
- throws ManifoldCFException
- {
- lockManager.enterWriteLock(trackerWriteLock);
- try
- {
- Long currentTime = readTime();
- String currentProcessID = readProcessID();
- String currentReproID = readReproID();
- if (currentTime == null || currentProcessID == null || currentReproID == null)
- return null;
- return currentTime;
- }
- finally
- {
- lockManager.leaveWriteLock(trackerWriteLock);
- }
- }
+ throws ManifoldCFException;
/** Complete a reprioritization activity. Prioritization will be marked as complete
* only if the processID matches the one that started the current reprioritization.
*@param processID is the process ID of the process completing the prioritization.
*/
public void doneReprioritization(String reproID)
- throws ManifoldCFException
- {
- lockManager.enterWriteLock(trackerWriteLock);
- try
- {
- Long currentTime = readTime();
- String currentProcessID = readProcessID();
- String currentReproID = readReproID();
- if (currentTime != null && currentProcessID != null && currentReproID != null && currentReproID.equals(reproID))
- {
- // Null out the fields
- writeTime(null);
- writeProcessID(null);
- writeReproID(null);
- }
- }
- finally
- {
- lockManager.leaveWriteLock(trackerWriteLock);
- }
- }
+ throws ManifoldCFException;
/** Check if the specified processID is the one performing reprioritization.
*@param processID is the process ID to check.
*@return the repro ID if the processID is confirmed to be the one.
*/
public String isSpecifiedProcessReprioritizing(String processID)
- throws ManifoldCFException
- {
- lockManager.enterWriteLock(trackerWriteLock);
- try
- {
- Long currentTime = readTime();
- String currentProcessID = readProcessID();
- String currentReproID = readReproID();
- if (currentTime != null && currentProcessID != null && currentReproID != null && currentProcessID.equals(processID))
- return currentReproID;
- return null;
- }
- finally
- {
- lockManager.leaveWriteLock(trackerWriteLock);
- }
- }
+ throws ManifoldCFException;
/** Assess the current minimum depth.
* This method is called to provide information about the priorities of the documents being currently
@@ -188,113 +68,30 @@ public class ReprioritizationTracker
*@param binNamesSet is the current set of priorities we see on the queuing operation.
*/
public void assessMinimumDepth(Double[] binNamesSet)
- throws ManifoldCFException
- {
- double newMinPriority = Double.MAX_VALUE;
- for (Double binValue : binNamesSet)
- {
- if (binValue.doubleValue() < newMinPriority)
- newMinPriority = binValue.doubleValue();
- }
-
- if (newMinPriority != Double.MAX_VALUE)
- {
-
- lockManager.enterWriteLock(trackerWriteLock);
- try
- {
- Long reproTime = readTime();
- String processID = readProcessID();
- if (reproTime == null || processID == null)
- {
- double currentMinimumDepth = readMinimumDepth();
-
- // Convert minPriority to minDepth.
- // Note that this calculation does not take into account anything having to do with connection rates, throttling,
- // or other adjustment factors. It allows us only to obtain the "raw" minimum depth: the depth without any
- // adjustments.
- double newMinDepth = Math.exp(newMinPriority)-1.0;
-
- if (newMinDepth > currentMinimumDepth)
- {
- writeMinimumDepth(newMinDepth);
- if (Logging.scheduling.isDebugEnabled())
- Logging.scheduling.debug("Setting new minimum depth value to "+new Double(currentMinimumDepth).toString());
- }
- else
- {
- if (newMinDepth < currentMinimumDepth && Logging.scheduling.isDebugEnabled())
- Logging.scheduling.debug("Minimum depth value seems to have been set too high too early! currentMin = "+new Double(currentMinimumDepth).toString()+"; queue value = "+new Double(newMinDepth).toString());
- }
- }
- }
- finally
- {
- lockManager.leaveWriteLock(trackerWriteLock);
- }
- }
- }
+ throws ManifoldCFException;
/** Retrieve current minimum depth.
*@return the current minimum depth to use.
*/
public double getMinimumDepth()
- throws ManifoldCFException
- {
- lockManager.enterReadLock(trackerWriteLock);
- try
- {
- return readMinimumDepth();
- }
- finally
- {
- lockManager.leaveReadLock(trackerWriteLock);
- }
- }
+ throws ManifoldCFException;
/** Note preload amounts.
*/
- public void addPreloadRequest(String binName, double weightedMinimumDepth)
- {
- PreloadRequest pr = preloadRequests.get(binName);
- if (pr == null)
- {
- pr = new PreloadRequest(weightedMinimumDepth);
- preloadRequests.put(binName,pr);
- }
- else
- pr.updateRequest(weightedMinimumDepth);
- }
-
+ public void addPreloadRequest(String binName, double weightedMinimumDepth);
/** Preload bin values. Call this OUTSIDE of a transaction.
*/
public void preloadBinValues()
- throws ManifoldCFException
- {
- for (String binName : preloadRequests.keySet())
- {
- PreloadRequest pr = preloadRequests.get(binName);
- double[] newValues = binManager.getIncrementBinValuesInTransaction(binName, pr.getWeightedMinimumDepth(), pr.getRequestCount());
- PreloadedValues pv = new PreloadedValues(newValues);
- preloadedValues.put(binName,pv);
- }
- preloadRequests.clear();
- }
+ throws ManifoldCFException;
/** Clear any preload requests.
*/
- public void clearPreloadRequests()
- {
- preloadRequests.clear();
- }
+ public void clearPreloadRequests();
/** Clear remaining preloaded values.
*/
- public void clearPreloadedValues()
- {
- preloadedValues.clear();
- }
+ public void clearPreloadedValues();
/** Get a bin value.
*@param binName is the bin name.
@@ -302,237 +99,7 @@ public class ReprioritizationTracker
*@return the bin value.
*/
public double getIncrementBinValue(String binName, double weightedMinimumDepth)
- throws ManifoldCFException
- {
- PreloadedValues pv = preloadedValues.get(binName);
- if (pv != null)
- {
- Double rval = pv.getNextValue();
- if (rval != null)
- return rval.doubleValue();
- }
- return binManager.getIncrementBinValues(binName, weightedMinimumDepth,1)[0];
- }
-
- // Protected methods
-
- /** Read time.
- *@return the time, or null if none.
- */
- protected Long readTime()
- throws ManifoldCFException
- {
- byte[] timeData = lockManager.readData(trackerTimestampResource);
- if (timeData == null || timeData.length != 8)
- return null;
-
- long rval = ((long)timeData[0]) & 0xffL +
- (((long)timeData[1]) << 8) & 0xff00L +
- (((long)timeData[2]) << 16) & 0xff0000L +
- (((long)timeData[3]) << 24) & 0xff000000L +
- (((long)timeData[4]) << 32) & 0xff00000000L +
- (((long)timeData[5]) << 40) & 0xff0000000000L +
- (((long)timeData[6]) << 48) & 0xff000000000000L +
- (((long)timeData[7]) << 56) & 0xff00000000000000L;
-
- return new Long(rval);
- }
-
- /** Write time.
- *@param time is the time to write.
- */
- protected void writeTime(Long timeValue)
- throws ManifoldCFException
- {
- if (timeValue == null)
- lockManager.writeData(trackerTimestampResource, null);
- else
- {
- long time = timeValue.longValue();
- byte[] timeData = new byte[8];
- timeData[0] = (byte)(time & 0xffL);
- timeData[1] = (byte)((time >> 8) & 0xffL);
- timeData[2] = (byte)((time >> 16) & 0xffL);
- timeData[3] = (byte)((time >> 24) & 0xffL);
- timeData[4] = (byte)((time >> 32) & 0xffL);
- timeData[5] = (byte)((time >> 40) & 0xffL);
- timeData[6] = (byte)((time >> 48) & 0xffL);
- timeData[7] = (byte)((time >> 56) & 0xffL);
- lockManager.writeData(trackerTimestampResource, timeData);
- }
- }
-
- /** Read process ID.
- *@return processID, or null if none.
- */
- protected String readProcessID()
- throws ManifoldCFException
- {
- byte[] processIDData = lockManager.readData(trackerProcessIDResource);
- if (processIDData == null)
- return null;
- try
- {
- return new String(processIDData, "utf-8");
- }
- catch (UnsupportedEncodingException e)
- {
- throw new RuntimeException(e.getMessage(),e);
- }
- }
-
- /** Write process ID.
- *@param processID is the process ID to write.
- */
- protected void writeProcessID(String processID)
- throws ManifoldCFException
- {
- if (processID == null)
- lockManager.writeData(trackerProcessIDResource, null);
- else
- {
- try
- {
- byte[] processIDData = processID.getBytes("utf-8");
- lockManager.writeData(trackerProcessIDResource, processIDData);
- }
- catch (UnsupportedEncodingException e)
- {
- throw new RuntimeException(e.getMessage(),e);
- }
- }
- }
-
- /** Read repriotization ID.
- *@return reproID, or null if none.
- */
- protected String readReproID()
- throws ManifoldCFException
- {
- byte[] reproIDData = lockManager.readData(trackerReproIDResource);
- if (reproIDData == null)
- return null;
- try
- {
- return new String(reproIDData, "utf-8");
- }
- catch (UnsupportedEncodingException e)
- {
- throw new RuntimeException(e.getMessage(),e);
- }
- }
-
- /** Write repro ID.
- *@param reproID is the repro ID to write.
- */
- protected void writeReproID(String reproID)
- throws ManifoldCFException
- {
- if (reproID == null)
- lockManager.writeData(trackerReproIDResource, null);
- else
- {
- try
- {
- byte[] reproIDData = reproID.getBytes("utf-8");
- lockManager.writeData(trackerReproIDResource, reproIDData);
- }
- catch (UnsupportedEncodingException e)
- {
- throw new RuntimeException(e.getMessage(),e);
- }
- }
- }
-
- /** Read minimum depth.
- *@return the minimum depth.
- */
- protected double readMinimumDepth()
- throws ManifoldCFException
- {
- byte[] data = lockManager.readData(trackerMinimumDepthResource);
- if (data == null || data.length != 8)
- return 0.0;
- long dataLong = ((long)data[0]) & 0xffL +
- (((long)data[1]) << 8) & 0xff00L +
- (((long)data[2]) << 16) & 0xff0000L +
- (((long)data[3]) << 24) & 0xff000000L +
- (((long)data[4]) << 32) & 0xff00000000L +
- (((long)data[5]) << 40) & 0xff0000000000L +
- (((long)data[6]) << 48) & 0xff000000000000L +
- (((long)data[7]) << 56) & 0xff00000000000000L;
-
- return Double.longBitsToDouble(dataLong);
- }
-
- /** Write minimum depth.
- *@param the minimum depth.
- */
- protected void writeMinimumDepth(double depth)
- throws ManifoldCFException
- {
- long dataLong = Double.doubleToLongBits(depth);
- byte[] data = new byte[8];
- data[0] = (byte)(dataLong & 0xffL);
- data[1] = (byte)((dataLong >> 8) & 0xffL);
- data[2] = (byte)((dataLong >> 16) & 0xffL);
- data[3] = (byte)((dataLong >> 24) & 0xffL);
- data[4] = (byte)((dataLong >> 32) & 0xffL);
- data[5] = (byte)((dataLong >> 40) & 0xffL);
- data[6] = (byte)((dataLong >> 48) & 0xffL);
- data[7] = (byte)((dataLong >> 56) & 0xffL);
- lockManager.writeData(trackerMinimumDepthResource,data);
- }
-
- /** A preload request */
- protected static class PreloadRequest
- {
- protected double weightedMinimumDepth;
- protected int requestCount;
-
- public PreloadRequest(double weightedMinimumDepth)
- {
- this.weightedMinimumDepth = weightedMinimumDepth;
- this.requestCount = 1;
- }
-
- public void updateRequest(double weightedMinimumDepth)
- {
- if (this.weightedMinimumDepth < weightedMinimumDepth)
- this.weightedMinimumDepth = weightedMinimumDepth;
- requestCount++;
- }
-
- public double getWeightedMinimumDepth()
- {
- return weightedMinimumDepth;
- }
-
- public int getRequestCount()
- {
- return requestCount;
- }
- }
-
- /** A set of preloaded values */
- protected static class PreloadedValues
- {
- protected double[] values;
- protected int valueIndex;
-
- public PreloadedValues(double[] values)
- {
- this.values = values;
- this.valueIndex = valueIndex;
- }
-
- public Double getNextValue()
- {
- if (valueIndex == values.length)
- return null;
- return new Double(values[valueIndex++]);
- }
- }
+ throws ManifoldCFException;
}
Added: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/ReprioritizationTrackerFactory.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/ReprioritizationTrackerFactory.java?rev=1549594&view=auto
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/ReprioritizationTrackerFactory.java (added)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/ReprioritizationTrackerFactory.java Mon Dec 9 15:09:02 2013
@@ -0,0 +1,53 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.interfaces;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.crawler.system.*;
+
+/** Factory class for IReprioritizationTracker.
+*/
+public class ReprioritizationTrackerFactory
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ // Name
+ protected final static String reprioritizationTrackerName = "_ReprioritizationTracker_";
+
+ private ReprioritizationTrackerFactory()
+ {
+ }
+
+ /** Create a reprioritization tracker handle.
+ *@param threadContext is the thread context.
+ *@return the handle.
+ */
+ public static IReprioritizationTracker make(IThreadContext threadContext)
+ throws ManifoldCFException
+ {
+ Object o = threadContext.get(reprioritizationTrackerName);
+ if (o == null || !(o instanceof IReprioritizationTracker))
+ {
+ o = new org.apache.manifoldcf.crawler.reprioritizationtracker.ReprioritizationTracker(threadContext);
+ threadContext.save(reprioritizationTrackerName,o);
+ }
+ return (IReprioritizationTracker)o;
+ }
+
+}
Propchange: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/ReprioritizationTrackerFactory.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/ReprioritizationTrackerFactory.java
------------------------------------------------------------------------------
svn:keywords = Id
Copied: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java (from r1549500, manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java)
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java?p2=manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java&p1=manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java&r1=1549500&r2=1549594&rev=1549594&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java Mon Dec 9 15:09:02 2013
@@ -16,11 +16,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.manifoldcf.crawler.system;
+package org.apache.manifoldcf.crawler.reprioritizationtracker;
import org.apache.manifoldcf.core.interfaces.*;
import org.apache.manifoldcf.agents.interfaces.*;
import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.Logging;
import java.util.*;
import java.io.*;
@@ -29,7 +30,7 @@ import java.io.*;
* and are completed if those processes die by the threads that clean up
* after the original process.
*/
-public class ReprioritizationTracker
+public class ReprioritizationTracker implements IReprioritizationTracker
{
public static final String _rcsid = "@(#)$Id$";
@@ -63,6 +64,7 @@ public class ReprioritizationTracker
* to complete.
*@param reproID is the reprocessing thread ID
*/
+ @Override
public void startReprioritization(long prioritizationTime, String processID, String reproID)
throws ManifoldCFException
{
@@ -111,6 +113,7 @@ public class ReprioritizationTracker
* performing any prioritization steps.
*@return the current prioritization timestamp, or null if no prioritization is in effect.
*/
+ @Override
public Long checkReprioritizationInProgress()
throws ManifoldCFException
{
@@ -134,6 +137,7 @@ public class ReprioritizationTracker
* only if the processID matches the one that started the current reprioritization.
*@param processID is the process ID of the process completing the prioritization.
*/
+ @Override
public void doneReprioritization(String reproID)
throws ManifoldCFException
{
@@ -161,6 +165,7 @@ public class ReprioritizationTracker
*@param processID is the process ID to check.
*@return the repro ID if the processID is confirmed to be the one.
*/
+ @Override
public String isSpecifiedProcessReprioritizing(String processID)
throws ManifoldCFException
{
@@ -187,6 +192,7 @@ public class ReprioritizationTracker
* adequately met.
*@param binNamesSet is the current set of priorities we see on the queuing operation.
*/
+ @Override
public void assessMinimumDepth(Double[] binNamesSet)
throws ManifoldCFException
{
@@ -238,6 +244,7 @@ public class ReprioritizationTracker
/** Retrieve current minimum depth.
*@return the current minimum depth to use.
*/
+ @Override
public double getMinimumDepth()
throws ManifoldCFException
{
@@ -254,6 +261,7 @@ public class ReprioritizationTracker
/** Note preload amounts.
*/
+ @Override
public void addPreloadRequest(String binName, double weightedMinimumDepth)
{
PreloadRequest pr = preloadRequests.get(binName);
@@ -269,6 +277,7 @@ public class ReprioritizationTracker
/** Preload bin values. Call this OUTSIDE of a transaction.
*/
+ @Override
public void preloadBinValues()
throws ManifoldCFException
{
@@ -284,6 +293,7 @@ public class ReprioritizationTracker
/** Clear any preload requests.
*/
+ @Override
public void clearPreloadRequests()
{
preloadRequests.clear();
@@ -291,6 +301,7 @@ public class ReprioritizationTracker
/** Clear remaining preloaded values.
*/
+ @Override
public void clearPreloadedValues()
{
preloadedValues.clear();
@@ -301,6 +312,7 @@ public class ReprioritizationTracker
*@param weightedMinimumDepth is the minimum depth to use.
*@return the bin value.
*/
+ @Override
public double getIncrementBinValue(String binName, double weightedMinimumDepth)
throws ManifoldCFException
{
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java?rev=1549594&r1=1549593&r2=1549594&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java Mon Dec 9 15:09:02 2013
@@ -169,7 +169,7 @@ public class CrawlerAgent implements IAg
{
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.cleanupProcessData(cleanupProcessID);
- ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
+ IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
String reproID = rt.isSpecifiedProcessReprioritizing(cleanupProcessID);
if (reproID != null)
{
@@ -204,7 +204,7 @@ public class CrawlerAgent implements IAg
break;
// Calculate new priorities for all these documents
- ManifoldCF.writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap,rt,updateTime);
+ ManifoldCF.writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap,updateTime);
Logging.threads.debug("Reprioritized "+Integer.toString(docs.length)+" not-yet-processed documents in "+new Long(System.currentTimeMillis()-startTime)+" ms");
}
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupThread.java?rev=1549594&r1=1549593&r2=1549594&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentCleanupThread.java Mon Dec 9 15:09:02 2013
@@ -80,7 +80,7 @@ public class DocumentCleanupThread exten
IIncrementalIngester ingester = IncrementalIngesterFactory.make(threadContext);
IJobManager jobManager = JobManagerFactory.make(threadContext);
IRepositoryConnectionManager connMgr = RepositoryConnectionManagerFactory.make(threadContext);
- ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
+ IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireThread.java?rev=1549594&r1=1549593&r2=1549594&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ExpireThread.java Mon Dec 9 15:09:02 2013
@@ -70,7 +70,7 @@ public class ExpireThread extends Thread
IIncrementalIngester ingester = IncrementalIngesterFactory.make(threadContext);
IJobManager jobManager = JobManagerFactory.make(threadContext);
IRepositoryConnectionManager connMgr = RepositoryConnectionManagerFactory.make(threadContext);
- ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
+ IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1549594&r1=1549593&r2=1549594&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java Mon Dec 9 15:09:02 2013
@@ -854,7 +854,7 @@ public class ManifoldCF extends org.apac
*/
public static void requeueDocumentsDueToCarrydown(IJobManager jobManager,
DocumentDescription[] requeueCandidates,
- IRepositoryConnector connector, IRepositoryConnection connection, ReprioritizationTracker rt, long currentTime)
+ IRepositoryConnector connector, IRepositoryConnection connection, IReprioritizationTracker rt, long currentTime)
throws ManifoldCFException
{
// A list of document descriptions from finishDocuments() above represents those documents that may need to be requeued, for the
@@ -925,7 +925,7 @@ public class ManifoldCF extends org.apac
ILockManager lockManager = LockManagerFactory.make(threadContext);
IJobManager jobManager = JobManagerFactory.make(threadContext);
IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(threadContext);
- ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
+ IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
String reproID = IDFactory.make(threadContext);
@@ -957,7 +957,7 @@ public class ManifoldCF extends org.apac
break;
// Calculate new priorities for all these documents
- writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap,rt,updateTime);
+ writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap,updateTime);
Logging.threads.debug("Reprioritized "+Integer.toString(docs.length)+" not-yet-processed documents in "+new Long(System.currentTimeMillis()-startTime)+" ms");
}
@@ -969,12 +969,13 @@ public class ManifoldCF extends org.apac
*/
public static void writeDocumentPriorities(IThreadContext threadContext, DocumentDescription[] descs,
Map<String,IRepositoryConnection> connectionMap, Map<Long,IJobDescription> jobDescriptionMap,
- ReprioritizationTracker rt, long currentTime)
+ long currentTime)
throws ManifoldCFException
{
IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(threadContext);
IJobManager jobManager = JobManagerFactory.make(threadContext);
+ IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
if (Logging.scheduling.isDebugEnabled())
Logging.scheduling.debug("Reprioritizing "+Integer.toString(descs.length)+" documents");
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PriorityCalculator.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PriorityCalculator.java?rev=1549594&r1=1549593&r2=1549594&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PriorityCalculator.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PriorityCalculator.java Mon Dec 9 15:09:02 2013
@@ -37,7 +37,7 @@ public class PriorityCalculator implemen
protected final IRepositoryConnection connection;
protected final String[] binNames;
- protected final ReprioritizationTracker rt;
+ protected final IReprioritizationTracker rt;
protected final double[] binCountScaleFactors;
protected final double[] weightedMinimumDepths;
@@ -45,7 +45,7 @@ public class PriorityCalculator implemen
protected Double cachedValue = null;
/** Constructor. */
- public PriorityCalculator(ReprioritizationTracker rt, IRepositoryConnection connection, String[] documentBins)
+ public PriorityCalculator(IReprioritizationTracker rt, IRepositoryConnection connection, String[] documentBins)
throws ManifoldCFException
{
this.connection = connection;
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingActivity.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingActivity.java?rev=1549594&r1=1549593&r2=1549594&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingActivity.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingActivity.java Mon Dec 9 15:09:02 2013
@@ -40,7 +40,7 @@ public class SeedingActivity implements
protected final String connectionName;
protected final IRepositoryConnectionManager connManager;
protected final IJobManager jobManager;
- protected final ReprioritizationTracker rt;
+ protected final IReprioritizationTracker rt;
protected final IRepositoryConnection connection;
protected final IRepositoryConnector connector;
protected final Long jobID;
@@ -59,7 +59,7 @@ public class SeedingActivity implements
*/
public SeedingActivity(String connectionName, IRepositoryConnectionManager connManager,
IJobManager jobManager,
- ReprioritizationTracker rt, IRepositoryConnection connection, IRepositoryConnector connector,
+ IReprioritizationTracker rt, IRepositoryConnection connection, IRepositoryConnector connector,
Long jobID, String[] legalLinkTypes, boolean overrideSchedule, int hopcountMethod, String processID)
{
this.processID = processID;
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java?rev=1549594&r1=1549593&r2=1549594&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java Mon Dec 9 15:09:02 2013
@@ -66,7 +66,7 @@ public class SeedingThread extends Threa
IThreadContext threadContext = ThreadContextFactory.make();
IJobManager jobManager = JobManagerFactory.make(threadContext);
IRepositoryConnectionManager connectionMgr = RepositoryConnectionManagerFactory.make(threadContext);
- ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
+ IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java?rev=1549594&r1=1549593&r2=1549594&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java Mon Dec 9 15:09:02 2013
@@ -68,7 +68,6 @@ public class SetPriorityThread extends T
IThreadContext threadContext = ThreadContextFactory.make();
IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(threadContext);
IJobManager jobManager = JobManagerFactory.make(threadContext);
- ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
Logging.threads.debug("Set priority thread coming up");
@@ -127,7 +126,7 @@ public class SetPriorityThread extends T
if (desc != null)
{
ManifoldCF.writeDocumentPriorities(threadContext,
- new DocumentDescription[]{desc},connectionMap,jobDescriptionMap,rt,currentTime);
+ new DocumentDescription[]{desc},connectionMap,jobDescriptionMap,currentTime);
processedCount++;
continue;
}
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java?rev=1549594&r1=1549593&r2=1549594&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java Mon Dec 9 15:09:02 2013
@@ -60,7 +60,7 @@ public class StartupThread extends Threa
IThreadContext threadContext = ThreadContextFactory.make();
IJobManager jobManager = JobManagerFactory.make(threadContext);
IRepositoryConnectionManager connectionMgr = RepositoryConnectionManagerFactory.make(threadContext);
- ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
+ IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java?rev=1549594&r1=1549593&r2=1549594&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java Mon Dec 9 15:09:02 2013
@@ -94,7 +94,7 @@ public class StufferThread extends Threa
IIncrementalIngester ingester = IncrementalIngesterFactory.make(threadContext);
IJobManager jobManager = JobManagerFactory.make(threadContext);
ILockManager lockManager = LockManagerFactory.make(threadContext);
- ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
+ IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1549594&r1=1549593&r2=1549594&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java Mon Dec 9 15:09:02 2013
@@ -77,7 +77,7 @@ public class WorkerThread extends Thread
IBinManager binManager = BinManagerFactory.make(threadContext);
IRepositoryConnectionManager connMgr = RepositoryConnectionManagerFactory.make(threadContext);
IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(threadContext);
- ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
+ IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
@@ -981,7 +981,7 @@ public class WorkerThread extends Thread
List<QueuedDocument> hopcountremoveList,
IIncrementalIngester ingester,
Long jobID, String[] legalLinkTypes, OutputActivity ingestLogger,
- int hopcountMethod, ReprioritizationTracker rt, long currentTime)
+ int hopcountMethod, IReprioritizationTracker rt, long currentTime)
throws ManifoldCFException
{
// Remove from index
@@ -1003,7 +1003,7 @@ public class WorkerThread extends Thread
List<QueuedDocument> deleteList,
IIncrementalIngester ingester,
Long jobID, String[] legalLinkTypes, OutputActivity ingestLogger,
- int hopcountMethod, ReprioritizationTracker rt, long currentTime)
+ int hopcountMethod, IReprioritizationTracker rt, long currentTime)
throws ManifoldCFException
{
// Remove from index
@@ -1090,7 +1090,7 @@ public class WorkerThread extends Thread
*/
protected static void processJobQueueDeletions(List<QueuedDocument> jobmanagerDeleteList,
IRepositoryConnector connector, IRepositoryConnection connection, IJobManager jobManager,
- Long jobID, String[] legalLinkTypes, int hopcountMethod, ReprioritizationTracker rt, long currentTime)
+ Long jobID, String[] legalLinkTypes, int hopcountMethod, IReprioritizationTracker rt, long currentTime)
throws ManifoldCFException
{
// Now, do the document queue cleanup for deletions.
@@ -1123,7 +1123,7 @@ public class WorkerThread extends Thread
*/
protected static void processJobQueueHopcountRemovals(List<QueuedDocument> jobmanagerRemovalList,
IRepositoryConnector connector, IRepositoryConnection connection, IJobManager jobManager,
- Long jobID, String[] legalLinkTypes, int hopcountMethod, ReprioritizationTracker rt, long currentTime)
+ Long jobID, String[] legalLinkTypes, int hopcountMethod, IReprioritizationTracker rt, long currentTime)
throws ManifoldCFException
{
// Now, do the document queue cleanup for deletions.
@@ -1446,7 +1446,7 @@ public class WorkerThread extends Thread
protected final IRepositoryConnectionManager connMgr;
protected final String[] legalLinkTypes;
protected final OutputActivity ingestLogger;
- protected final ReprioritizationTracker rt;
+ protected final IReprioritizationTracker rt;
protected final HashMap abortSet;
protected final String outputVersion;
protected final String parameterVersion;
@@ -1468,7 +1468,7 @@ public class WorkerThread extends Thread
*@param ingester is the ingester
*/
public ProcessActivity(String processID, IThreadContext threadContext,
- ReprioritizationTracker rt, IJobManager jobManager,
+ IReprioritizationTracker rt, IJobManager jobManager,
IIncrementalIngester ingester, long currentTime,
IJobDescription job, IRepositoryConnection connection, IRepositoryConnector connector,
IRepositoryConnectionManager connMgr, String[] legalLinkTypes, OutputActivity ingestLogger,