You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/11/23 17:55:12 UTC

svn commit: r1641222 - in /manifoldcf/trunk: CHANGES.txt framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java

Author: kwright
Date: Sun Nov 23 16:55:12 2014
New Revision: 1641222

URL: http://svn.apache.org/r1641222
Log:
Fix for CONNECTORS-1112.

Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1641222&r1=1641221&r2=1641222&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Sun Nov 23 16:55:12 2014
@@ -3,6 +3,10 @@ $Id$
 
 ======================= 2.0-dev =====================
 
+CONNECTORS-1112: Reduce contention for zookeeper resources by
+caching local configuration data, and by reorganizing stuffer thread.
+(Aeham Abushwashi, Karl Wright)
+
 CONNECTORS-1111: Component cleanup did not work properly.
 (Markus Schuch, Karl Wright)
 

Modified: manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java?rev=1641222&r1=1641221&r2=1641222&view=diff
==============================================================================
--- manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java (original)
+++ manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java Sun Nov 23 16:55:12 2014
@@ -57,6 +57,9 @@ public class ZooKeeperLockManager extend
   protected static Integer ephemeralPoolLocker = new Integer(0);
   protected static ZooKeeperEphemeralNodePool myEphemeralNodes = null;
 
+  // Cached local values
+  protected ManifoldCFConfiguration cachedConfiguration = null;
+  
   /** Constructor */
   public ZooKeeperLockManager()
     throws ManifoldCFException
@@ -660,27 +663,31 @@ public class ZooKeeperLockManager extend
   public ManifoldCFConfiguration getSharedConfiguration()
     throws ManifoldCFException
   {
-    try
+    if (cachedConfiguration == null)
     {
-      ZooKeeperConnection connection = pool.grab();
       try
       {
-        // Read as a byte array, then parse
-        byte[] configurationData = connection.readData(CONFIGURATION_PATH);
-        if (configurationData != null)
-          return new ManifoldCFConfiguration(new ByteArrayInputStream(configurationData));
-        else
-          return new ManifoldCFConfiguration();
+        ZooKeeperConnection connection = pool.grab();
+        try
+        {
+          // Read as a byte array, then parse
+          byte[] configurationData = connection.readData(CONFIGURATION_PATH);
+          if (configurationData != null)
+            cachedConfiguration = new ManifoldCFConfiguration(new ByteArrayInputStream(configurationData));
+          else
+            cachedConfiguration = new ManifoldCFConfiguration();
+        }
+        finally
+        {
+          pool.release(connection);
+        }
       }
-      finally
+      catch (InterruptedException e)
       {
-        pool.release(connection);
+        throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
     }
-    catch (InterruptedException e)
-    {
-      throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-    }
+    return cachedConfiguration;
   }
 
   /** Write shared configuration.  Caller closes the input stream.

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java?rev=1641222&r1=1641221&r2=1641222&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java Sun Nov 23 16:55:12 2014
@@ -224,6 +224,7 @@ public class StufferThread extends Threa
           String[] documentIDHashes = new String[descs.length];
 
           // Go through the documents and set up jobs, prefixed id's
+          Set<String> connectionNames = new HashSet<String>();
           for (int i = 0; i < descs.length; i++)
           {
             DocumentDescription dd = descs[i];
@@ -235,6 +236,7 @@ public class StufferThread extends Threa
             }
             jobs[i] = job;
             String connectionName = job.getConnectionName();
+            connectionNames.add(connectionName);
             documentClasses[i] = connectionName;
             pipelineSpecifications[i] = new PipelineSpecificationBasic(job);
             IRepositoryConnection connection = connectionMap.get(connectionName);
@@ -266,74 +268,102 @@ public class StufferThread extends Threa
           // We need to go through the list, and segregate them by job, so the individual
           // connectors can work in batch.
           documentSets.clear();
-          for (int i = 0; i < descs.length; i++)
+
+          // Prepare to grab all the connector instances we'll need
+          String[] orderingKeys = new String[connectionNames.size()];
+          IRepositoryConnection[] grabConnections = new IRepositoryConnection[connectionNames.size()];
+          int z = 0;
+          for (String connectionName : connectionNames)
           {
-            Long jobID = jobs[i].getID();
+            orderingKeys[z] = connectionName;
+            IRepositoryConnection connection = connectionMap.get(connectionName);
+            grabConnections[z] = connection;
+            z++;
+          }
 
-            // We have to see how we are doing with respect to the limit for this connector.
-            // We also need to log the queuing activity to the queue tracker, so that
-            // the priority setter thread can do its thing properly.
-
-            // Get a repository connection appropriate for this document.
-            IRepositoryConnection connection = connections[i];
-            int maxDocuments;
-            String[] binNames;
+          String[][] descBinNames = new String[descs.length][];
+          int[] descMaxDocuments = new int[descs.length];
+          try
+          {
+            IRepositoryConnector[] connectors = repositoryConnectorPool.grabMultiple(orderingKeys,grabConnections);
             try
             {
-              // Grab a connector handle
-              IRepositoryConnector connector = repositoryConnectorPool.grab(connection);
-              if (connector == null)
+              // Map from connection name to connector instance
+              Map<String,IRepositoryConnector> connectorMap = new HashMap<String,IRepositoryConnector>();
+              for (z = 0; z < orderingKeys.length; z++)
               {
-                maxDocuments = 1;
-                binNames = new String[]{""};
+                connectorMap.put(orderingKeys[z],connectors[z]);
               }
-              else
+
+              for (int i = 0; i < descs.length; i++)
               {
-                try
+                // We have to see how we are doing with respect to the limit for this connector.
+                // We also need to log the queuing activity to the queue tracker, so that
+                // the priority setter thread can do its thing properly.
+
+                // Get a repository connection appropriate for this document.
+                IRepositoryConnection connection = connections[i];
+                int maxDocuments;
+                String[] binNames;
+                // Grab a connector handle
+                IRepositoryConnector connector = connectorMap.get(connection.getName());
+                if (connector == null)
+                {
+                  maxDocuments = 1;
+                  binNames = new String[]{""};
+                }
+                else
                 {
                   // Convert the document identifier to a URI
                   maxDocuments = connector.getMaxDocumentRequest();
                   // Get the bins for the document identifier
                   binNames = connector.getBinNames(descs[i].getDocumentIdentifier());
                 }
-                finally
-                {
-                  repositoryConnectorPool.release(connection,connector);
-                }
+                descBinNames[i] = binNames;
+                descMaxDocuments[i] = maxDocuments;
               }
             }
-            catch (ManifoldCFException e)
+            finally
             {
-              // If we were interrupted, then we are allowed to leave, because the process is terminating, but that's the only exception to the rule
-              if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
-                throw e;
-
-              // Note: We really should never leave this block by throwing an exception, since that could easily leave dangling
-              // active jobqueue entries around.  Instead, log the error and continue IN ALL CASES.
-              Logging.threads.error("Stuffer thread ManifoldCF Exception thrown: "+e.getMessage()+" - continuing",
-                e);
-              maxDocuments = 1;
-              binNames = new String[]{""};
+              // Release all the connector instances we grabbed
+              repositoryConnectorPool.releaseMultiple(grabConnections,connectors);
             }
-            catch (OutOfMemoryError e)
-            {
-              System.err.println("agents process ran out of memory - shutting down");
-              e.printStackTrace(System.err);
-              System.exit(-200);
-              // These are needed because the compiler doesn't know System.exit ends the thread.
-              maxDocuments = 1;
+            
+          }
+          catch (ManifoldCFException e)
+          {
+            // If we were interrupted, then we are allowed to leave, because the process is terminating, but that's the only exception to the rule
+            if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+              throw e;
+
+            // Note: We really should never leave this block by throwing an exception, since that could easily leave dangling
+            // active jobqueue entries around.  Instead, log the error and continue IN ALL CASES.
+            Logging.threads.error("Stuffer thread ManifoldCF Exception thrown: "+e.getMessage()+" - continuing",
+              e);
+          }
+          catch (OutOfMemoryError e)
+          {
+            System.err.println("agents process ran out of memory - shutting down");
+            e.printStackTrace(System.err);
+            System.exit(-200);
+          }
+          catch (Throwable e)
+          {
+            // Note: We really should never leave this block by throwing an exception, since that could easily leave dangling
+            // active jobqueue entries around.  Instead, log the error and continue IN ALL CASES.
+            Logging.threads.fatal("Stuffer thread Throwable thrown: "+e.getMessage()+" - continuing",
+              e);
+          }
+
+          for (int i = 0; i < descs.length; i++)
+          {
+            Long jobID = jobs[i].getID();
+            String[] binNames = descBinNames[i];
+            if (binNames == null)
               binNames = new String[]{""};
-            }
-            catch (Throwable e)
-            {
-              // Note: We really should never leave this block by throwing an exception, since that could easily leave dangling
-              // active jobqueue entries around.  Instead, log the error and continue IN ALL CASES.
-              Logging.threads.fatal("Stuffer thread Throwable thrown: "+e.getMessage()+" - continuing",
-                e);
+            int maxDocuments = descMaxDocuments[i];
+            if (maxDocuments == 0)
               maxDocuments = 1;
-              binNames = new String[]{""};
-            }
-
             QueuedDocument qd = new QueuedDocument(descs[i],(Map<String,DocumentIngestStatusSet>)versions[i],binNames);
 
             // Grab the arraylist that's there, or create it.