You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/12/05 11:31:02 UTC
svn commit: r1548081 [3/3] - in /manifoldcf/trunk: ./
framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/
framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/
framework/agents/src/main/java/org/apache/manif...
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1548081&r1=1548080&r2=1548081&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java Thu Dec 5 10:31:01 2013
@@ -105,6 +105,15 @@ public class ManifoldCF extends org.apac
public static void localCleanup(IThreadContext tc)
{
+ try
+ {
+ RepositoryConnectorPoolFactory.make(tc).closeAllConnectors();
+ }
+ catch (ManifoldCFException e)
+ {
+ if (Logging.root != null)
+ Logging.root.warn("Exception tossed on repository connector pool cleanup: "+e.getMessage(),e);
+ }
}
/** Create system database using superuser properties from properties.xml.
@@ -948,8 +957,7 @@ public class ManifoldCF extends org.apac
break;
// Calculate new priorities for all these documents
- writeDocumentPriorities(threadContext,connectionManager,jobManager,docs,connectionMap,jobDescriptionMap,
- rt,updateTime);
+ writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap,rt,updateTime);
Logging.threads.debug("Reprioritized "+Integer.toString(docs.length)+" not-yet-processed documents in "+new Long(System.currentTimeMillis()-startTime)+" ms");
}
@@ -959,12 +967,15 @@ public class ManifoldCF extends org.apac
/** Write a set of document priorities, based on the current queue tracker.
*/
- public static void writeDocumentPriorities(IThreadContext threadContext, IRepositoryConnectionManager mgr,
- IJobManager jobManager, DocumentDescription[] descs,
+ public static void writeDocumentPriorities(IThreadContext threadContext, DocumentDescription[] descs,
Map<String,IRepositoryConnection> connectionMap, Map<Long,IJobDescription> jobDescriptionMap,
ReprioritizationTracker rt, long currentTime)
throws ManifoldCFException
{
+ IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
+ IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(threadContext);
+ IJobManager jobManager = JobManagerFactory.make(threadContext);
+
if (Logging.scheduling.isDebugEnabled())
Logging.scheduling.debug("Reprioritizing "+Integer.toString(descs.length)+" documents");
@@ -992,10 +1003,7 @@ public class ManifoldCF extends org.apac
String[] binNames;
// Grab a connector handle
- IRepositoryConnector connector = RepositoryConnectorFactory.grab(threadContext,
- connection.getClassName(),
- connection.getConfigParams(),
- connection.getMaxConnections());
+ IRepositoryConnector connector = repositoryConnectorPool.grab(connection);
try
{
if (connector == null)
@@ -1006,7 +1014,7 @@ public class ManifoldCF extends org.apac
}
finally
{
- RepositoryConnectorFactory.release(connector);
+ repositoryConnectorPool.release(connector);
}
priorities[i] = new PriorityCalculator(rt,connection,binNames);
@@ -1217,6 +1225,7 @@ public class ManifoldCF extends org.apac
{
try
{
+ IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(tc);
IOutputConnectionManager connectionManager = OutputConnectionManagerFactory.make(tc);
IOutputConnection connection = connectionManager.load(connectionName);
if (connection == null)
@@ -1227,7 +1236,7 @@ public class ManifoldCF extends org.apac
String results;
// Grab a connection handle, and call the test method
- IOutputConnector connector = OutputConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
+ IOutputConnector connector = outputConnectorPool.grab(connection);
try
{
results = connector.check();
@@ -1238,7 +1247,7 @@ public class ManifoldCF extends org.apac
}
finally
{
- OutputConnectorFactory.release(connector);
+ outputConnectorPool.release(connector);
}
ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE);
@@ -1258,6 +1267,7 @@ public class ManifoldCF extends org.apac
{
try
{
+ IAuthorityConnectorPool authorityConnectorPool = AuthorityConnectorPoolFactory.make(tc);
IAuthorityConnectionManager connectionManager = AuthorityConnectionManagerFactory.make(tc);
IAuthorityConnection connection = connectionManager.load(connectionName);
if (connection == null)
@@ -1268,7 +1278,7 @@ public class ManifoldCF extends org.apac
String results;
// Grab a connection handle, and call the test method
- IAuthorityConnector connector = AuthorityConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
+ IAuthorityConnector connector = authorityConnectorPool.grab(connection);
try
{
results = connector.check();
@@ -1279,7 +1289,7 @@ public class ManifoldCF extends org.apac
}
finally
{
- AuthorityConnectorFactory.release(connector);
+ authorityConnectorPool.release(connector);
}
ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE);
@@ -1299,6 +1309,7 @@ public class ManifoldCF extends org.apac
{
try
{
+ IMappingConnectorPool mappingConnectorPool = MappingConnectorPoolFactory.make(tc);
IMappingConnectionManager connectionManager = MappingConnectionManagerFactory.make(tc);
IMappingConnection connection = connectionManager.load(connectionName);
if (connection == null)
@@ -1309,7 +1320,7 @@ public class ManifoldCF extends org.apac
String results;
// Grab a connection handle, and call the test method
- IMappingConnector connector = MappingConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
+ IMappingConnector connector = mappingConnectorPool.grab(connection);
try
{
results = connector.check();
@@ -1320,7 +1331,7 @@ public class ManifoldCF extends org.apac
}
finally
{
- MappingConnectorFactory.release(connector);
+ mappingConnectorPool.release(connector);
}
ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE);
@@ -1340,6 +1351,7 @@ public class ManifoldCF extends org.apac
{
try
{
+ IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(tc);
IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(tc);
IRepositoryConnection connection = connectionManager.load(connectionName);
if (connection == null)
@@ -1350,7 +1362,7 @@ public class ManifoldCF extends org.apac
String results;
// Grab a connection handle, and call the test method
- IRepositoryConnector connector = RepositoryConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
+ IRepositoryConnector connector = repositoryConnectorPool.grab(connection);
try
{
results = connector.check();
@@ -1361,7 +1373,7 @@ public class ManifoldCF extends org.apac
}
finally
{
- RepositoryConnectorFactory.release(connector);
+ repositoryConnectorPool.release(connector);
}
ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE);
@@ -1382,6 +1394,7 @@ public class ManifoldCF extends org.apac
{
try
{
+ IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(tc);
IOutputConnectionManager connectionManager = OutputConnectionManagerFactory.make(tc);
IOutputConnection connection = connectionManager.load(connectionName);
if (connection == null)
@@ -1391,14 +1404,14 @@ public class ManifoldCF extends org.apac
}
// Grab a connection handle, and call the test method
- IOutputConnector connector = OutputConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
+ IOutputConnector connector = outputConnectorPool.grab(connection);
try
{
return connector.requestInfo(output,command)?READRESULT_FOUND:READRESULT_NOTFOUND;
}
finally
{
- OutputConnectorFactory.release(connector);
+ outputConnectorPool.release(connector);
}
}
catch (ManifoldCFException e)
@@ -1414,6 +1427,7 @@ public class ManifoldCF extends org.apac
{
try
{
+ IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(tc);
IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(tc);
IRepositoryConnection connection = connectionManager.load(connectionName);
if (connection == null)
@@ -1423,14 +1437,14 @@ public class ManifoldCF extends org.apac
}
// Grab a connection handle, and call the test method
- IRepositoryConnector connector = RepositoryConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
+ IRepositoryConnector connector = repositoryConnectorPool.grab(connection);
try
{
return connector.requestInfo(output,command)?READRESULT_FOUND:READRESULT_NOTFOUND;
}
finally
{
- RepositoryConnectorFactory.release(connector);
+ repositoryConnectorPool.release(connector);
}
}
catch (ManifoldCFException e)
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java Thu Dec 5 10:31:01 2013
@@ -68,11 +68,8 @@ public class SeedingThread extends Threa
IRepositoryConnectionManager connectionMgr = RepositoryConnectionManagerFactory.make(threadContext);
ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
- IDBInterface database = DBInterfaceFactory.make(threadContext,
- ManifoldCF.getMasterDatabaseName(),
- ManifoldCF.getMasterDatabaseUsername(),
- ManifoldCF.getMasterDatabasePassword());
-
+ IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
+
String[] identifiers = new String[MAX_COUNT];
// Loop
while (true)
@@ -126,17 +123,12 @@ public class SeedingThread extends Threa
int hopcountMethod = jobDescription.getHopcountMode();
IRepositoryConnection connection = connectionMgr.load(jobDescription.getConnectionName());
- IRepositoryConnector connector = RepositoryConnectorFactory.grab(threadContext,
- connection.getClassName(),
- connection.getConfigParams(),
- connection.getMaxConnections());
+ IRepositoryConnector connector = repositoryConnectorPool.grab(connection);
// Null will come back if the connector instance could not be obtained, so just skip in that case.
if (connector == null)
continue;
try
{
-
-
// Get the number of link types.
String[] legalLinkTypes = connector.getRelationshipTypes();
@@ -174,7 +166,7 @@ public class SeedingThread extends Threa
}
finally
{
- RepositoryConnectorFactory.release(connector);
+ repositoryConnectorPool.release(connector);
}
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java Thu Dec 5 10:31:01 2013
@@ -126,7 +126,7 @@ public class SetPriorityThread extends T
DocumentDescription desc = blockingDocuments.getBlockingDocument();
if (desc != null)
{
- ManifoldCF.writeDocumentPriorities(threadContext,mgr,jobManager,
+ ManifoldCF.writeDocumentPriorities(threadContext,
new DocumentDescription[]{desc},connectionMap,jobDescriptionMap,rt,currentTime);
processedCount++;
continue;
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java Thu Dec 5 10:31:01 2013
@@ -62,11 +62,8 @@ public class StartupThread extends Threa
IRepositoryConnectionManager connectionMgr = RepositoryConnectionManagerFactory.make(threadContext);
ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
- IDBInterface database = DBInterfaceFactory.make(threadContext,
- ManifoldCF.getMasterDatabaseName(),
- ManifoldCF.getMasterDatabaseUsername(),
- ManifoldCF.getMasterDatabasePassword());
-
+ IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
+
// Loop
while (true)
{
@@ -117,22 +114,19 @@ public class StartupThread extends Threa
int hopcountMethod = jobDescription.getHopcountMode();
IRepositoryConnection connection = connectionMgr.load(jobDescription.getConnectionName());
- IRepositoryConnector connector = RepositoryConnectorFactory.grab(threadContext,
- connection.getClassName(),
- connection.getConfigParams(),
- connection.getMaxConnections());
+ IRepositoryConnector connector = repositoryConnectorPool.grab(connection);
// If the attempt to grab a connector instance failed, don't start the job, of course.
if (connector == null)
continue;
- // Only now record the fact that we are trying to start the job.
- connectionMgr.recordHistory(jobDescription.getConnectionName(),
- null,connectionMgr.ACTIVITY_JOBSTART,null,
- jobID.toString()+"("+jobDescription.getDescription()+")",null,null,null);
-
try
{
+ // Only now record the fact that we are trying to start the job.
+ connectionMgr.recordHistory(jobDescription.getConnectionName(),
+ null,connectionMgr.ACTIVITY_JOBSTART,null,
+ jobID.toString()+"("+jobDescription.getDescription()+")",null,null,null);
+
int model = connector.getConnectorModel();
// Get the number of link types.
String[] legalLinkTypes = connector.getRelationshipTypes();
@@ -175,7 +169,7 @@ public class StartupThread extends Threa
}
finally
{
- RepositoryConnectorFactory.release(connector);
+ repositoryConnectorPool.release(connector);
}
// Start this job!
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java Thu Dec 5 10:31:01 2013
@@ -89,6 +89,8 @@ public class StufferThread extends Threa
IJobManager jobManager = JobManagerFactory.make(threadContext);
ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
+ IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
+
Logging.threads.debug("Stuffer thread: Low water mark is "+Integer.toString(lowWaterMark)+"; amount per stuffing is "+Integer.toString(stuffAmt));
// This is used to adjust the number of records returned for jobs
@@ -250,10 +252,7 @@ public class StufferThread extends Threa
try
{
// Grab a connector handle
- IRepositoryConnector connector = RepositoryConnectorFactory.grab(threadContext,
- connection.getClassName(),
- connection.getConfigParams(),
- connection.getMaxConnections());
+ IRepositoryConnector connector = repositoryConnectorPool.grab(connection);
if (connector == null)
{
maxDocuments = 1;
@@ -270,7 +269,7 @@ public class StufferThread extends Threa
}
finally
{
- RepositoryConnectorFactory.release(connector);
+ repositoryConnectorPool.release(connector);
}
}
}
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java Thu Dec 5 10:31:01 2013
@@ -79,6 +79,8 @@ public class WorkerThread extends Thread
IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(threadContext);
ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
+ IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
+
List<DocumentToProcess> fetchList = new ArrayList<DocumentToProcess>();
Map<String,String> versionMap = new HashMap<String,String>();
List<QueuedDocument> finishList = new ArrayList<QueuedDocument>();
@@ -257,10 +259,7 @@ public class WorkerThread extends Thread
IRepositoryConnector connector = null;
if (activeDocuments.size() > 0 || hopcountremoveList.size() > 0)
{
- connector = RepositoryConnectorFactory.grab(threadContext,
- connection.getClassName(),
- connection.getConfigParams(),
- connection.getMaxConnections());
+ connector = repositoryConnectorPool.grab(connection);
// If we wind up with a null here, it means that a document got queued for a connector which is now gone.
// Basically, what we want to do in that case is to treat this kind of like a service interruption - the document
@@ -832,7 +831,7 @@ public class WorkerThread extends Thread
}
finally
{
- RepositoryConnectorFactory.release(connector);
+ repositoryConnectorPool.release(connector);
}
}