You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/11/23 17:55:12 UTC
svn commit: r1641222 - in /manifoldcf/trunk: CHANGES.txt
framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java
framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
Author: kwright
Date: Sun Nov 23 16:55:12 2014
New Revision: 1641222
URL: http://svn.apache.org/r1641222
Log:
Fix for CONNECTORS-1112.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1641222&r1=1641221&r2=1641222&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Sun Nov 23 16:55:12 2014
@@ -3,6 +3,10 @@ $Id$
======================= 2.0-dev =====================
+CONNECTORS-1112: Reduce contention for zookeeper resources by
+caching local configuration data, and by reorganizing stuffer thread.
+(Aeham Abushwashi, Karl Wright)
+
CONNECTORS-1111: Component cleanup did not work properly.
(Markus Schuch, Karl Wright)
Modified: manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java?rev=1641222&r1=1641221&r2=1641222&view=diff
==============================================================================
--- manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java (original)
+++ manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java Sun Nov 23 16:55:12 2014
@@ -57,6 +57,9 @@ public class ZooKeeperLockManager extend
protected static Integer ephemeralPoolLocker = new Integer(0);
protected static ZooKeeperEphemeralNodePool myEphemeralNodes = null;
+ // Cached local values
+ protected ManifoldCFConfiguration cachedConfiguration = null;
+
/** Constructor */
public ZooKeeperLockManager()
throws ManifoldCFException
@@ -660,27 +663,31 @@ public class ZooKeeperLockManager extend
public ManifoldCFConfiguration getSharedConfiguration()
throws ManifoldCFException
{
- try
+ if (cachedConfiguration == null)
{
- ZooKeeperConnection connection = pool.grab();
try
{
- // Read as a byte array, then parse
- byte[] configurationData = connection.readData(CONFIGURATION_PATH);
- if (configurationData != null)
- return new ManifoldCFConfiguration(new ByteArrayInputStream(configurationData));
- else
- return new ManifoldCFConfiguration();
+ ZooKeeperConnection connection = pool.grab();
+ try
+ {
+ // Read as a byte array, then parse
+ byte[] configurationData = connection.readData(CONFIGURATION_PATH);
+ if (configurationData != null)
+ cachedConfiguration = new ManifoldCFConfiguration(new ByteArrayInputStream(configurationData));
+ else
+ cachedConfiguration = new ManifoldCFConfiguration();
+ }
+ finally
+ {
+ pool.release(connection);
+ }
}
- finally
+ catch (InterruptedException e)
{
- pool.release(connection);
+ throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
}
}
- catch (InterruptedException e)
- {
- throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
+ return cachedConfiguration;
}
/** Write shared configuration. Caller closes the input stream.
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java?rev=1641222&r1=1641221&r2=1641222&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java Sun Nov 23 16:55:12 2014
@@ -224,6 +224,7 @@ public class StufferThread extends Threa
String[] documentIDHashes = new String[descs.length];
// Go through the documents and set up jobs, prefixed id's
+ Set<String> connectionNames = new HashSet<String>();
for (int i = 0; i < descs.length; i++)
{
DocumentDescription dd = descs[i];
@@ -235,6 +236,7 @@ public class StufferThread extends Threa
}
jobs[i] = job;
String connectionName = job.getConnectionName();
+ connectionNames.add(connectionName);
documentClasses[i] = connectionName;
pipelineSpecifications[i] = new PipelineSpecificationBasic(job);
IRepositoryConnection connection = connectionMap.get(connectionName);
@@ -266,74 +268,102 @@ public class StufferThread extends Threa
// We need to go through the list, and segregate them by job, so the individual
// connectors can work in batch.
documentSets.clear();
- for (int i = 0; i < descs.length; i++)
+
+ // Prepare to grab all the connector instances we'll need
+ String[] orderingKeys = new String[connectionNames.size()];
+ IRepositoryConnection[] grabConnections = new IRepositoryConnection[connectionNames.size()];
+ int z = 0;
+ for (String connectionName : connectionNames)
{
- Long jobID = jobs[i].getID();
+ orderingKeys[z] = connectionName;
+ IRepositoryConnection connection = connectionMap.get(connectionName);
+ grabConnections[z] = connection;
+ z++;
+ }
- // We have to see how we are doing with respect to the limit for this connector.
- // We also need to log the queuing activity to the queue tracker, so that
- // the priority setter thread can do its thing properly.
-
- // Get a repository connection appropriate for this document.
- IRepositoryConnection connection = connections[i];
- int maxDocuments;
- String[] binNames;
+ String[][] descBinNames = new String[descs.length][];
+ int[] descMaxDocuments = new int[descs.length];
+ try
+ {
+ IRepositoryConnector[] connectors = repositoryConnectorPool.grabMultiple(orderingKeys,grabConnections);
try
{
- // Grab a connector handle
- IRepositoryConnector connector = repositoryConnectorPool.grab(connection);
- if (connector == null)
+ // Map from connection name to connector instance
+ Map<String,IRepositoryConnector> connectorMap = new HashMap<String,IRepositoryConnector>();
+ for (z = 0; z < orderingKeys.length; z++)
{
- maxDocuments = 1;
- binNames = new String[]{""};
+ connectorMap.put(orderingKeys[z],connectors[z]);
}
- else
+
+ for (int i = 0; i < descs.length; i++)
{
- try
+ // We have to see how we are doing with respect to the limit for this connector.
+ // We also need to log the queuing activity to the queue tracker, so that
+ // the priority setter thread can do its thing properly.
+
+ // Get a repository connection appropriate for this document.
+ IRepositoryConnection connection = connections[i];
+ int maxDocuments;
+ String[] binNames;
+ // Grab a connector handle
+ IRepositoryConnector connector = connectorMap.get(connection.getName());
+ if (connector == null)
+ {
+ maxDocuments = 1;
+ binNames = new String[]{""};
+ }
+ else
{
// Convert the document identifier to a URI
maxDocuments = connector.getMaxDocumentRequest();
// Get the bins for the document identifier
binNames = connector.getBinNames(descs[i].getDocumentIdentifier());
}
- finally
- {
- repositoryConnectorPool.release(connection,connector);
- }
+ descBinNames[i] = binNames;
+ descMaxDocuments[i] = maxDocuments;
}
}
- catch (ManifoldCFException e)
+ finally
{
- // If we were interrupted, then we are allowed to leave, because the process is terminating, but that's the only exception to the rule
- if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
- throw e;
-
- // Note: We really should never leave this block by throwing an exception, since that could easily leave dangling
- // active jobqueue entries around. Instead, log the error and continue IN ALL CASES.
- Logging.threads.error("Stuffer thread ManifoldCF Exception thrown: "+e.getMessage()+" - continuing",
- e);
- maxDocuments = 1;
- binNames = new String[]{""};
+ // Release all the connector instances we grabbed
+ repositoryConnectorPool.releaseMultiple(grabConnections,connectors);
}
- catch (OutOfMemoryError e)
- {
- System.err.println("agents process ran out of memory - shutting down");
- e.printStackTrace(System.err);
- System.exit(-200);
- // These are needed because the compiler doesn't know System.exit ends the thread.
- maxDocuments = 1;
+
+ }
+ catch (ManifoldCFException e)
+ {
+ // If we were interrupted, then we are allowed to leave, because the process is terminating, but that's the only exception to the rule
+ if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+ throw e;
+
+ // Note: We really should never leave this block by throwing an exception, since that could easily leave dangling
+ // active jobqueue entries around. Instead, log the error and continue IN ALL CASES.
+ Logging.threads.error("Stuffer thread ManifoldCF Exception thrown: "+e.getMessage()+" - continuing",
+ e);
+ }
+ catch (OutOfMemoryError e)
+ {
+ System.err.println("agents process ran out of memory - shutting down");
+ e.printStackTrace(System.err);
+ System.exit(-200);
+ }
+ catch (Throwable e)
+ {
+ // Note: We really should never leave this block by throwing an exception, since that could easily leave dangling
+ // active jobqueue entries around. Instead, log the error and continue IN ALL CASES.
+ Logging.threads.fatal("Stuffer thread Throwable thrown: "+e.getMessage()+" - continuing",
+ e);
+ }
+
+ for (int i = 0; i < descs.length; i++)
+ {
+ Long jobID = jobs[i].getID();
+ String[] binNames = descBinNames[i];
+ if (binNames == null)
binNames = new String[]{""};
- }
- catch (Throwable e)
- {
- // Note: We really should never leave this block by throwing an exception, since that could easily leave dangling
- // active jobqueue entries around. Instead, log the error and continue IN ALL CASES.
- Logging.threads.fatal("Stuffer thread Throwable thrown: "+e.getMessage()+" - continuing",
- e);
+ int maxDocuments = descMaxDocuments[i];
+ if (maxDocuments == 0)
maxDocuments = 1;
- binNames = new String[]{""};
- }
-
QueuedDocument qd = new QueuedDocument(descs[i],(Map<String,DocumentIngestStatusSet>)versions[i],binNames);
// Grab the arraylist that's there, or create it.