You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2015/12/05 13:36:30 UTC
svn commit: r1718072 - in /manifoldcf/trunk/framework: ./
pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/
pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/
pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritiz...
Author: kwright
Date: Sat Dec 5 12:36:29 2015
New Revision: 1718072
URL: http://svn.apache.org/viewvc?rev=1718072&view=rev
Log:
Separate bins by connector class name. Part of CONNECTORS-1249.
Modified:
manifoldcf/trunk/framework/build.xml
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PriorityCalculator.java
Modified: manifoldcf/trunk/framework/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/build.xml?rev=1718072&r1=1718071&r2=1718072&view=diff
==============================================================================
--- manifoldcf/trunk/framework/build.xml (original)
+++ manifoldcf/trunk/framework/build.xml Sat Dec 5 12:36:29 2015
@@ -1833,17 +1833,19 @@
</junit>
</target>
- <target name="run-pull-agent-tests" depends="compile-pull-agent,compile-pull-agent-tests">
+ <target name="run-pull-agent-tests" depends="compile-pull-agent,compile-connector-common,compile-pull-agent-tests">
<mkdir dir="test-output"/>
<junit fork="true" maxmemory="128m" dir="test-output" outputtoformatters="true" showoutput="true" haltonfailure="true">
<classpath>
<path refid="framework-classpath"/>
<pathelement location="build/core/classes"/>
<pathelement location="build/core-tests/classes"/>
+ <pathelement location="build/ui-core/classes"/>
<pathelement location="build/agents/classes"/>
<pathelement location="build/agents-tests/classes"/>
<pathelement location="build/pull-agent/classes"/>
<pathelement location="build/pull-agent-tests/classes"/>
+ <pathelement location="build/connector-common/classes"/>
</classpath>
<formatter type="brief" usefile="false"/>
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java?rev=1718072&r1=1718071&r2=1718072&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java Sat Dec 5 12:36:29 2015
@@ -45,6 +45,7 @@ public class BinManager extends org.apac
public static final String _rcsid = "@(#)$Id$";
// Field names
+ public final static String connectorClassField = "connectorclass";
public final static String binNameField = "binname";
public final static String binCounterField = "bincounter";
@@ -71,17 +72,23 @@ public class BinManager extends org.apac
{
HashMap map = new HashMap();
// HSQLDB does not like null primary keys!!
+ map.put(connectorClassField,new ColumnDescription("VARCHAR(255)",false,true,null,null,false));
map.put(binNameField,new ColumnDescription("VARCHAR(255)",false,true,null,null,false));
map.put(binCounterField,new ColumnDescription("FLOAT",false,false,null,null,false));
performCreate(map,null);
}
else
{
- // Upgrade goes here if needed
+ // Upgrade
+ if (existing.get(connectorClassField) == null) {
+ HashMap map = new HashMap();
+ map.put(connectorClassField,new ColumnDescription("VARCHAR(255)",false,true,null,null,false));
+ performAlter(map,null,null,null);
+ }
}
// Index management goes here
- IndexDescription binIndex = new IndexDescription(true,new String[]{binNameField});
+ IndexDescription binIndex = new IndexDescription(true,new String[]{connectorClassField,binNameField});
// Get rid of indexes that shouldn't be there
Map indexes = getTableIndexes(null,null);
@@ -125,19 +132,22 @@ public class BinManager extends org.apac
/** Get N bin values (and set next one). If the record does not yet exist, create it with a starting value.
* We expect this to happen within a transaction!!.
+ *@param connectorClass is the class name of the connector
*@param binName is the name of the bin (256 char max)
*@param newBinValue is the value to use if there is no such bin yet. This is the value that will be
* returned; what will be stored will be that value + 1.
*@param count is the number of values desired.
*@return the counter values.
*/
- public double[] getIncrementBinValues(String binName, double newBinValue, int count)
+ @Override
+ public double[] getIncrementBinValues(String connectorClass, String binName, double newBinValue, int count)
throws ManifoldCFException
{
double[] returnValues = new double[count];
// SELECT FOR UPDATE/MODIFY is the most common path
ArrayList params = new ArrayList();
String query = buildConjunctionClause(params,new ClauseDescription[]{
+ new UnitaryClause(connectorClassField,connectorClass),
new UnitaryClause(binNameField,binName)});
IResultSet result = performQuery("SELECT "+binCounterField+" FROM "+getTableName()+" WHERE "+query+" FOR UPDATE",params,null,null);
if (result.getRowCount() > 0)
@@ -165,6 +175,7 @@ public class BinManager extends org.apac
newBinValue += 1.0;
}
HashMap map = new HashMap();
+ map.put(connectorClassField,connectorClass);
map.put(binNameField,binName);
map.put(binCounterField,new Double(newBinValue));
performInsert(map,null);
@@ -174,6 +185,7 @@ public class BinManager extends org.apac
/** Get N bin values (and set next one). If the record does not yet exist, create it with a starting value.
* This method invokes its own retry-able transaction.
+ *@param connectorClass is the class name of the connector
*@param binName is the name of the bin (256 char max)
*@param newBinValue is the value to use if there is no such bin yet. This is the value that will be
* returned; what will be stored will be that value + 1.
@@ -181,7 +193,7 @@ public class BinManager extends org.apac
*@return the counter values.
*/
@Override
- public double[] getIncrementBinValuesInTransaction(String binName, double newBinValue, int count)
+ public double[] getIncrementBinValuesInTransaction(String connectorClass, String binName, double newBinValue, int count)
throws ManifoldCFException
{
while (true)
@@ -190,7 +202,7 @@ public class BinManager extends org.apac
beginTransaction();
try
{
- return getIncrementBinValues(binName, newBinValue, count);
+ return getIncrementBinValues(connectorClass, binName, newBinValue, count);
}
catch (Error e)
{
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java?rev=1718072&r1=1718071&r2=1718072&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java Sat Dec 5 12:36:29 2015
@@ -43,24 +43,26 @@ public interface IBinManager
/** Get N bin values (and set next one). If the record does not yet exist, create it with a starting value.
* We expect this to happen within a transaction!!
+ *@param connectorClass is the class name of the connector
*@param binName is the name of the bin (256 char max)
*@param newBinValue is the value to use if there is no such bin yet. This is the value that will be
* returned; what will be stored will be that value + 1.
*@param count is the number of values desired.
*@return the counter values.
*/
- public double[] getIncrementBinValues(String binName, double newBinValue, int count)
+ public double[] getIncrementBinValues(String connectorClass, String binName, double newBinValue, int count)
throws ManifoldCFException;
/** Get N bin values (and set next one). If the record does not yet exist, create it with a starting value.
* This method invokes its own retry-able transaction.
+ *@param connectorClass is the class name of the connector
*@param binName is the name of the bin (256 char max)
*@param newBinValue is the value to use if there is no such bin yet. This is the value that will be
* returned; what will be stored will be that value + 1.
*@param count is the number of values desired.
*@return the counter values.
*/
- public double[] getIncrementBinValuesInTransaction(String binName, double newBinValue, int count)
+ public double[] getIncrementBinValuesInTransaction(String connectorClass, String binName, double newBinValue, int count)
throws ManifoldCFException;
}
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java?rev=1718072&r1=1718071&r2=1718072&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java Sat Dec 5 12:36:29 2015
@@ -70,7 +70,7 @@ public interface IReprioritizationTracke
/** Note preload amounts.
*/
- public void addPreloadRequest(String binName, double weightedMinimumDepth);
+ public void addPreloadRequest(String connectorClass, String binName, double weightedMinimumDepth);
/** Preload bin values. Call this OUTSIDE of a transaction.
*/
@@ -86,11 +86,12 @@ public interface IReprioritizationTracke
public void clearPreloadedValues();
/** Get a bin value. Must be called INSIDE a transaction.
+ *@param connectorClass is the connector class name.
*@param binName is the bin name.
*@param weightedMinimumDepth is the minimum depth to use.
*@return the bin value.
*/
- public double getIncrementBinValue(String binName, double weightedMinimumDepth)
+ public double getIncrementBinValue(String connectorClass, String binName, double weightedMinimumDepth)
throws ManifoldCFException;
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java?rev=1718072&r1=1718071&r2=1718072&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java Sat Dec 5 12:36:29 2015
@@ -46,9 +46,9 @@ public class ReprioritizationTracker imp
protected final IBinManager binManager;
/** Preload requests */
- protected final Map<String,PreloadRequest> preloadRequests = new HashMap<String,PreloadRequest>();
+ protected final Map<PreloadKey,PreloadRequest> preloadRequests = new HashMap<PreloadKey,PreloadRequest>();
/** Preload values */
- protected final Map<String,PreloadedValues> preloadedValues = new HashMap<String,PreloadedValues>();
+ protected final Map<PreloadKey,PreloadedValues> preloadedValues = new HashMap<PreloadKey,PreloadedValues>();
/** Constructor.
*/
@@ -231,13 +231,14 @@ public class ReprioritizationTracker imp
/** Note preload amounts.
*/
@Override
- public void addPreloadRequest(String binName, double weightedMinimumDepth)
+ public void addPreloadRequest(String connectorClass, String binName, double weightedMinimumDepth)
{
- PreloadRequest pr = preloadRequests.get(binName);
+ final PreloadKey pk = new PreloadKey(connectorClass, binName);
+ PreloadRequest pr = preloadRequests.get(pk);
if (pr == null)
{
pr = new PreloadRequest(weightedMinimumDepth);
- preloadRequests.put(binName,pr);
+ preloadRequests.put(pk,pr);
}
else
pr.updateRequest(weightedMinimumDepth);
@@ -250,12 +251,12 @@ public class ReprioritizationTracker imp
public void preloadBinValues()
throws ManifoldCFException
{
- for (String binName : preloadRequests.keySet())
+ for (PreloadKey pk : preloadRequests.keySet())
{
- PreloadRequest pr = preloadRequests.get(binName);
- double[] newValues = binManager.getIncrementBinValuesInTransaction(binName, pr.getWeightedMinimumDepth(), pr.getRequestCount());
+ PreloadRequest pr = preloadRequests.get(pk);
+ double[] newValues = binManager.getIncrementBinValuesInTransaction(pk.connectorClass, pk.binName, pr.getWeightedMinimumDepth(), pr.getRequestCount());
PreloadedValues pv = new PreloadedValues(newValues);
- preloadedValues.put(binName,pv);
+ preloadedValues.put(pk,pv);
}
preloadRequests.clear();
}
@@ -277,22 +278,24 @@ public class ReprioritizationTracker imp
}
/** Get a bin value.
+ *@param connectorClass is the connector class name.
*@param binName is the bin name.
*@param weightedMinimumDepth is the minimum depth to use.
*@return the bin value.
*/
@Override
- public double getIncrementBinValue(String binName, double weightedMinimumDepth)
+ public double getIncrementBinValue(String connectorClass, String binName, double weightedMinimumDepth)
throws ManifoldCFException
{
- PreloadedValues pv = preloadedValues.get(binName);
+ final PreloadKey key = new PreloadKey(connectorClass,binName);
+ PreloadedValues pv = preloadedValues.get(key);
if (pv != null)
{
Double rval = pv.getNextValue();
if (rval != null)
return rval.doubleValue();
}
- return binManager.getIncrementBinValues(binName, weightedMinimumDepth,1)[0];
+ return binManager.getIncrementBinValues(connectorClass, binName, weightedMinimumDepth,1)[0];
}
// Protected methods
@@ -445,6 +448,29 @@ public class ReprioritizationTracker imp
}
}
-
+ /** Connector class name, bin name pair */
+ protected static class PreloadKey
+ {
+ public final String connectorClass;
+ public final String binName;
+
+ public PreloadKey(final String connectorClass, final String binName) {
+ this.connectorClass = connectorClass;
+ this.binName = binName;
+ }
+
+ public int hashCode() {
+ return connectorClass.hashCode() + binName.hashCode();
+ }
+
+ public boolean equals(final Object o) {
+ if (!(o instanceof PreloadKey))
+ return false;
+ final PreloadKey pk = (PreloadKey)o;
+ return connectorClass.equals(pk.connectorClass) &&
+ binName.equals(pk.binName);
+ }
+ }
+
}
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PriorityCalculator.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PriorityCalculator.java?rev=1718072&r1=1718071&r2=1718072&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PriorityCalculator.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/PriorityCalculator.java Sat Dec 5 12:36:29 2015
@@ -120,7 +120,7 @@ public class PriorityCalculator implemen
for (int i = 0; i < binNames.length; i++)
{
String binName = binNames[i];
- rt.addPreloadRequest(binName, weightedMinimumDepths[i]);
+ rt.addPreloadRequest(connection.getClassName(), binName, weightedMinimumDepths[i]);
}
}
@@ -148,7 +148,7 @@ public class PriorityCalculator implemen
double binCountScaleFactor = binCountScaleFactors[i];
double weightedMinimumDepth = weightedMinimumDepths[i];
- double thisCount = rt.getIncrementBinValue(binName,weightedMinimumDepth);
+ double thisCount = rt.getIncrementBinValue(connection.getClassName(),binName,weightedMinimumDepth);
double adjustedCount;
// Use the scale factor already calculated above to yield a priority that is adjusted for the fetch rate.
if (binCountScaleFactor == Double.POSITIVE_INFINITY)