You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/11/26 20:57:27 UTC

svn commit: r1641911 - in /manifoldcf/trunk: ./ framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/ framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/ framework/pull-agent/src/main/java/org/a...

Author: kwright
Date: Wed Nov 26 19:57:26 2014
New Revision: 1641911

URL: http://svn.apache.org/r1641911
Log:
Fix for CONNECTORS-1116.

Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
    manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1641911&r1=1641910&r2=1641911&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Wed Nov 26 19:57:26 2014
@@ -3,6 +3,10 @@ $Id$
 
 ======================= 2.0-dev =====================
 
+CONNECTORS-1116: Prevent requirement of unlimited handles for
+cached loads by limiting number of cached loads to 200 at a time.
+(Adrian Conlon, Karl Wright)
+
 CONNECTORS-1115: Add ability to retain all components of a document,
 so that individual ones do not need to be specified.
 (Markus Schuch, Karl Wright)

Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java?rev=1641911&r1=1641910&r2=1641911&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java (original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java Wed Nov 26 19:57:26 2014
@@ -196,12 +196,10 @@ public class OutputConnectionManager ext
       IResultSet set = performQuery("SELECT "+nameField+",lower("+nameField+") AS sortfield FROM "+getTableName()+" ORDER BY sortfield ASC",null,
         localCacheKeys,null);
       String[] names = new String[set.getRowCount()];
-      int i = 0;
-      while (i < names.length)
+      for (int i = 0; i < names.length; i++)
       {
         IResultRow row = set.getRow(i);
         names[i] = row.getValue(nameField).toString();
-        i++;
       }
       return loadMultiple(names);
     }
@@ -231,6 +229,8 @@ public class OutputConnectionManager ext
     return loadMultiple(new String[]{name})[0];
   }
 
+  protected final static int FETCH_MAX = 200;
+
   /** Load multiple output connections by name.
   *@param names are the names to load.
   *@return the loaded connection objects.
@@ -238,21 +238,46 @@ public class OutputConnectionManager ext
   public IOutputConnection[] loadMultiple(String[] names)
     throws ManifoldCFException
   {
+    IOutputConnection[] rval = new IOutputConnection[names.length];
+    if (names.length == 0)
+      return rval;
+    List<String> fetchNames = new ArrayList<String>();
+    int outputIndex = 0;
+    for (String name : names)
+    {
+      if (fetchNames.size() == FETCH_MAX)
+      {
+        outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
+        fetchNames.clear();
+      }
+      fetchNames.add(name);
+    }
+    loadMultipleInternal(rval,outputIndex,fetchNames);
+    return rval;
+  }
+  
+  protected int loadMultipleInternal(IOutputConnection[] rval, int outputIndex, List<String> fetchNames)
+    throws ManifoldCFException
+  {
     // Build description objects
-    OutputConnectionDescription[] objectDescriptions = new OutputConnectionDescription[names.length];
-    int i = 0;
+    OutputConnectionDescription[] objectDescriptions = new OutputConnectionDescription[fetchNames.size()];
     StringSetBuffer ssb = new StringSetBuffer();
-    while (i < names.length)
+    for (int i = 0; i < fetchNames.size(); i++)
     {
       ssb.clear();
-      ssb.add(getOutputConnectionKey(names[i]));
-      objectDescriptions[i] = new OutputConnectionDescription(names[i],new StringSet(ssb));
-      i++;
+      String name = fetchNames.get(i);
+      ssb.add(getOutputConnectionKey(name));
+      objectDescriptions[i] = new OutputConnectionDescription(name,new StringSet(ssb));
     }
 
     OutputConnectionExecutor exec = new OutputConnectionExecutor(this,objectDescriptions);
     cacheManager.findObjectsAndExecute(objectDescriptions,null,exec,getTransactionID());
-    return exec.getResults();
+    IOutputConnection[] results = exec.getResults();
+    for (IOutputConnection result : results)
+    {
+      rval[outputIndex++] = result;
+    }
+    return outputIndex;
   }
 
   /** Create a new output connection object.

Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java?rev=1641911&r1=1641910&r2=1641911&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java (original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java Wed Nov 26 19:57:26 2014
@@ -196,12 +196,10 @@ public class TransformationConnectionMan
       IResultSet set = performQuery("SELECT "+nameField+",lower("+nameField+") AS sortfield FROM "+getTableName()+" ORDER BY sortfield ASC",null,
         localCacheKeys,null);
       String[] names = new String[set.getRowCount()];
-      int i = 0;
-      while (i < names.length)
+      for (int i = 0; i < names.length; i++)
       {
         IResultRow row = set.getRow(i);
         names[i] = row.getValue(nameField).toString();
-        i++;
       }
       return loadMultiple(names);
     }
@@ -231,6 +229,8 @@ public class TransformationConnectionMan
     return loadMultiple(new String[]{name})[0];
   }
 
+  protected final static int FETCH_MAX = 200;
+
   /** Load multiple transformation connections by name.
   *@param names are the names to load.
   *@return the loaded connection objects.
@@ -238,21 +238,47 @@ public class TransformationConnectionMan
   public ITransformationConnection[] loadMultiple(String[] names)
     throws ManifoldCFException
   {
+    ITransformationConnection[] rval = new ITransformationConnection[names.length];
+    if (names.length == 0)
+      return rval;
+    List<String> fetchNames = new ArrayList<String>();
+    int outputIndex = 0;
+    for (String name : names)
+    {
+      if (fetchNames.size() == FETCH_MAX)
+      {
+        outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
+        fetchNames.clear();
+      }
+      fetchNames.add(name);
+    }
+    loadMultipleInternal(rval,outputIndex,fetchNames);
+    return rval;
+  }
+  
+  protected int loadMultipleInternal(ITransformationConnection[] rval, int outputIndex, List<String> fetchNames)
+    throws ManifoldCFException
+  {
+
     // Build description objects
-    TransformationConnectionDescription[] objectDescriptions = new TransformationConnectionDescription[names.length];
-    int i = 0;
+    TransformationConnectionDescription[] objectDescriptions = new TransformationConnectionDescription[fetchNames.size()];
     StringSetBuffer ssb = new StringSetBuffer();
-    while (i < names.length)
+    for (int i = 0; i < fetchNames.size(); i++)
     {
       ssb.clear();
-      ssb.add(getTransformationConnectionKey(names[i]));
-      objectDescriptions[i] = new TransformationConnectionDescription(names[i],new StringSet(ssb));
-      i++;
+      String name = fetchNames.get(i);
+      ssb.add(getTransformationConnectionKey(name));
+      objectDescriptions[i] = new TransformationConnectionDescription(name,new StringSet(ssb));
     }
 
     TransformationConnectionExecutor exec = new TransformationConnectionExecutor(this,objectDescriptions);
     cacheManager.findObjectsAndExecute(objectDescriptions,null,exec,getTransactionID());
-    return exec.getResults();
+    ITransformationConnection[] results = exec.getResults();
+    for (ITransformationConnection result : results)
+    {
+      rval[outputIndex++] = result;
+    }
+    return outputIndex;
   }
 
   /** Create a new transformation connection object.

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java?rev=1641911&r1=1641910&r2=1641911&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java Wed Nov 26 19:57:26 2014
@@ -249,12 +249,10 @@ public class AuthorityConnectionManager 
       sb.append(buildConjunctionClause(list,new ClauseDescription[]{new UnitaryClause(authDomainField,authDomain)}));
       IResultSet set = performQuery(sb.toString(),list,localCacheKeys,null);
       String[] names = new String[set.getRowCount()];
-      int i = 0;
-      while (i < names.length)
+      for (int i = 0; i < names.length; i++)
       {
         IResultRow row = set.getRow(i);
         names[i] = row.getValue(nameField).toString();
-        i++;
       }
       return loadMultiple(names);
     }
@@ -291,12 +289,10 @@ public class AuthorityConnectionManager 
       IResultSet set = performQuery("SELECT "+nameField+",lower("+nameField+") AS sortfield FROM "+getTableName()+" ORDER BY sortfield ASC",null,
         localCacheKeys,null);
       String[] names = new String[set.getRowCount()];
-      int i = 0;
-      while (i < names.length)
+      for (int i = 0; i < names.length; i++)
       {
         IResultRow row = set.getRow(i);
         names[i] = row.getValue(nameField).toString();
-        i++;
       }
       return loadMultiple(names);
     }
@@ -327,6 +323,8 @@ public class AuthorityConnectionManager 
     return loadMultiple(new String[]{name})[0];
   }
 
+  protected final static int FETCH_MAX = 200;
+
   /** Load multiple repository connections by name.
   *@param names are the names to load.
   *@return the loaded connection objects.
@@ -335,21 +333,46 @@ public class AuthorityConnectionManager 
   public IAuthorityConnection[] loadMultiple(String[] names)
     throws ManifoldCFException
   {
+    IAuthorityConnection[] rval = new IAuthorityConnection[names.length];
+    if (names.length == 0)
+      return rval;
+    List<String> fetchNames = new ArrayList<String>();
+    int outputIndex = 0;
+    for (String name : names)
+    {
+      if (fetchNames.size() == FETCH_MAX)
+      {
+        outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
+        fetchNames.clear();
+      }
+      fetchNames.add(name);
+    }
+    loadMultipleInternal(rval,outputIndex,fetchNames);
+    return rval;
+  }
+  
+  protected int loadMultipleInternal(IAuthorityConnection[] rval, int outputIndex, List<String> fetchNames)
+    throws ManifoldCFException
+  {
     // Build description objects
-    AuthorityConnectionDescription[] objectDescriptions = new AuthorityConnectionDescription[names.length];
-    int i = 0;
+    AuthorityConnectionDescription[] objectDescriptions = new AuthorityConnectionDescription[fetchNames.size()];
     StringSetBuffer ssb = new StringSetBuffer();
-    while (i < names.length)
+    for (int i = 0; i < fetchNames.size(); i++)
     {
       ssb.clear();
-      ssb.add(getAuthorityConnectionKey(names[i]));
-      objectDescriptions[i] = new AuthorityConnectionDescription(names[i],new StringSet(ssb));
-      i++;
+      String name = fetchNames.get(i);
+      ssb.add(getAuthorityConnectionKey(name));
+      objectDescriptions[i] = new AuthorityConnectionDescription(name,new StringSet(ssb));
     }
 
     AuthorityConnectionExecutor exec = new AuthorityConnectionExecutor(this,objectDescriptions);
     cacheManager.findObjectsAndExecute(objectDescriptions,null,exec,getTransactionID());
-    return exec.getResults();
+    IAuthorityConnection[] results = exec.getResults();
+    for (IAuthorityConnection result : results)
+    {
+      rval[outputIndex++] = result;
+    }
+    return outputIndex;
   }
 
   /** Create a new repository connection object.

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java?rev=1641911&r1=1641910&r2=1641911&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java Wed Nov 26 19:57:26 2014
@@ -298,12 +298,10 @@ public class MappingConnectionManager ex
       IResultSet set = performQuery("SELECT "+nameField+",lower("+nameField+") AS sortfield FROM "+getTableName()+" ORDER BY sortfield ASC",null,
         localCacheKeys,null);
       String[] names = new String[set.getRowCount()];
-      int i = 0;
-      while (i < names.length)
+      for (int i = 0; i < names.length; i++)
       {
         IResultRow row = set.getRow(i);
         names[i] = row.getValue(nameField).toString();
-        i++;
       }
       return loadMultiple(names);
     }
@@ -334,6 +332,8 @@ public class MappingConnectionManager ex
     return loadMultiple(new String[]{name})[0];
   }
 
+  protected final static int FETCH_MAX = 200;
+
   /** Load multiple mapping connections by name.
   *@param names are the names to load.
   *@return the loaded connection objects.
@@ -342,21 +342,46 @@ public class MappingConnectionManager ex
   public IMappingConnection[] loadMultiple(String[] names)
     throws ManifoldCFException
   {
+    IMappingConnection[] rval = new IMappingConnection[names.length];
+    if (names.length == 0)
+      return rval;
+    List<String> fetchNames = new ArrayList<String>();
+    int outputIndex = 0;
+    for (String name : names)
+    {
+      if (fetchNames.size() == FETCH_MAX)
+      {
+        outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
+        fetchNames.clear();
+      }
+      fetchNames.add(name);
+    }
+    loadMultipleInternal(rval,outputIndex,fetchNames);
+    return rval;
+  }
+  
+  protected int loadMultipleInternal(IMappingConnection[] rval, int outputIndex, List<String> fetchNames)
+    throws ManifoldCFException
+  {
     // Build description objects
-    MappingConnectionDescription[] objectDescriptions = new MappingConnectionDescription[names.length];
-    int i = 0;
+    MappingConnectionDescription[] objectDescriptions = new MappingConnectionDescription[fetchNames.size()];
     StringSetBuffer ssb = new StringSetBuffer();
-    while (i < names.length)
+    for (int i = 0; i < fetchNames.size(); i++)
     {
       ssb.clear();
-      ssb.add(getMappingConnectionKey(names[i]));
-      objectDescriptions[i] = new MappingConnectionDescription(names[i],new StringSet(ssb));
-      i++;
+      String name = fetchNames.get(i);
+      ssb.add(getMappingConnectionKey(name));
+      objectDescriptions[i] = new MappingConnectionDescription(name,new StringSet(ssb));
     }
 
     MappingConnectionExecutor exec = new MappingConnectionExecutor(this,objectDescriptions);
     cacheManager.findObjectsAndExecute(objectDescriptions,null,exec,getTransactionID());
-    return exec.getResults();
+    IMappingConnection[] results = exec.getResults();
+    for (IMappingConnection result : results)
+    {
+      rval[outputIndex++] = result;
+    }
+    return outputIndex;
   }
 
   /** Create a new repository connection object.

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java?rev=1641911&r1=1641910&r2=1641911&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java Wed Nov 26 19:57:26 2014
@@ -647,6 +647,9 @@ public class Jobs extends org.apache.man
     return rval;
   }
 
+  // Only fetch 200 jobs at a time, for resource reasons
+  protected final int MAX_FETCH = 200;
+
   /** Get a list of all jobs which are not in the process of being deleted already.
   *@return the array of all jobs.
   */
@@ -671,15 +674,15 @@ public class Jobs extends org.apache.man
       IResultSet set = performQuery("SELECT "+idField+","+descriptionField+" FROM "+
         getTableName()+" WHERE "+statusField+"!=? AND "+statusField+"!=? AND "+statusField+"!=? AND "+statusField+"!=?"+
         " ORDER BY "+descriptionField+" ASC",list,cacheKeys,null);
-      // Convert to an array of id's, and then load them
+      
       Long[] ids = new Long[set.getRowCount()];
       boolean[] readOnlies = new boolean[set.getRowCount()];
-      int i = 0;
-      while (i < ids.length)
+      
+      for (int i = 0; i < set.getRowCount(); i++)
       {
         IResultRow row = set.getRow(i);
         ids[i] = (Long)row.getValue(idField);
-        readOnlies[i++] = true;
+        readOnlies[i] = true;
       }
       return loadMultiple(ids,readOnlies);
     }
@@ -819,21 +822,50 @@ public class Jobs extends org.apache.man
   public IJobDescription[] loadMultiple(Long[] ids, boolean[] readOnlies)
     throws ManifoldCFException
   {
+    IJobDescription[] rval = new IJobDescription[ids.length];
+    if (ids.length == 0)
+      return rval;
+    
+    List<Long> idsToDo = new ArrayList<Long>();
+    List<Boolean> readOnliesToDo = new ArrayList<Boolean>();
+    int outputIndex = 0;
+    for (int i = 0; i < ids.length; i++)
+    {
+      if (idsToDo.size() == MAX_FETCH)
+      {
+        outputIndex = loadMultipleInternal(rval,outputIndex,idsToDo,readOnliesToDo);
+        idsToDo.clear();
+        readOnliesToDo.clear();
+      }
+      idsToDo.add(ids[i]);
+      readOnliesToDo.add(readOnlies[i]);
+    }
+    loadMultipleInternal(rval,outputIndex,idsToDo,readOnliesToDo);
+    return rval;
+  }
+
+  protected int loadMultipleInternal(IJobDescription[] rval, int outputIndex, List<Long> ids, List<Boolean> readOnlies)
+    throws ManifoldCFException
+  {
     // Build description objects
-    JobObjectDescription[] objectDescriptions = new JobObjectDescription[ids.length];
-    int i = 0;
+    JobObjectDescription[] objectDescriptions = new JobObjectDescription[ids.size()];
     StringSetBuffer ssb = new StringSetBuffer();
-    while (i < ids.length)
+    for (int i = 0; i < ids.size(); i++)
     {
+      Long id = ids.get(i);
       ssb.clear();
-      ssb.add(getJobIDKey(ids[i]));
-      objectDescriptions[i] = new JobObjectDescription(ids[i],new StringSet(ssb));
-      i++;
+      ssb.add(getJobIDKey(id));
+      objectDescriptions[i] = new JobObjectDescription(id,new StringSet(ssb));
     }
 
     JobObjectExecutor exec = new JobObjectExecutor(this,objectDescriptions);
     cacheManager.findObjectsAndExecute(objectDescriptions,null,exec,getTransactionID());
-    return exec.getResults(readOnlies);
+    IJobDescription[] results = exec.getResults(readOnlies);
+    for (IJobDescription result : results)
+    {
+      rval[outputIndex++] = result;
+    }
+    return outputIndex;
   }
 
   /** Save a job description.
@@ -3584,18 +3616,16 @@ public class Jobs extends org.apache.man
     /** Get the result.
     *@return the looked-up or read cached instances.
     */
-    public JobDescription[] getResults(boolean[] readOnlies)
+    public JobDescription[] getResults(List<Boolean> readOnlies)
     {
       JobDescription[] rval = new JobDescription[returnValues.length];
-      int i = 0;
-      while (i < rval.length)
+      for (int i = 0; i < rval.length; i++)
       {
         JobDescription jd = returnValues[i];
         if (jd != null)
-          rval[i] = jd.duplicate(readOnlies[i]);
+          rval[i] = jd.duplicate(readOnlies.get(i));
         else
           rval[i] = null;
-        i++;
       }
       return rval;
     }

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java?rev=1641911&r1=1641910&r2=1641911&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java Wed Nov 26 19:57:26 2014
@@ -266,12 +266,10 @@ public class RepositoryConnectionManager
       IResultSet set = performQuery("SELECT "+nameField+",lower("+nameField+") AS sortfield FROM "+getTableName()+" ORDER BY sortfield ASC",null,
         localCacheKeys,null);
       String[] names = new String[set.getRowCount()];
-      int i = 0;
-      while (i < names.length)
+      for (int i = 0; i < names.length; i++)
       {
         IResultRow row = set.getRow(i);
         names[i] = row.getValue(nameField).toString();
-        i++;
       }
       return loadMultiple(names);
     }
@@ -301,6 +299,8 @@ public class RepositoryConnectionManager
     return loadMultiple(new String[]{name})[0];
   }
 
+  protected final static int FETCH_MAX = 200;
+  
   /** Load multiple repository connections by name.
   *@param names are the names to load.
   *@return the loaded connection objects.
@@ -308,21 +308,46 @@ public class RepositoryConnectionManager
   public IRepositoryConnection[] loadMultiple(String[] names)
     throws ManifoldCFException
   {
+    IRepositoryConnection[] rval = new IRepositoryConnection[names.length];
+    if (names.length == 0)
+      return rval;
+    List<String> fetchNames = new ArrayList<String>();
+    int outputIndex = 0;
+    for (String name : names)
+    {
+      if (fetchNames.size() == FETCH_MAX)
+      {
+        outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
+        fetchNames.clear();
+      }
+      fetchNames.add(name);
+    }
+    loadMultipleInternal(rval,outputIndex,fetchNames);
+    return rval;
+  }
+  
+  protected int loadMultipleInternal(IRepositoryConnection[] rval, int outputIndex, List<String> fetchNames)
+    throws ManifoldCFException
+  {
     // Build description objects
-    RepositoryConnectionDescription[] objectDescriptions = new RepositoryConnectionDescription[names.length];
-    int i = 0;
+    RepositoryConnectionDescription[] objectDescriptions = new RepositoryConnectionDescription[fetchNames.size()];
     StringSetBuffer ssb = new StringSetBuffer();
-    while (i < names.length)
+    for (int i = 0; i < fetchNames.size(); i++)
     {
       ssb.clear();
-      ssb.add(getRepositoryConnectionKey(names[i]));
-      objectDescriptions[i] = new RepositoryConnectionDescription(names[i],new StringSet(ssb));
-      i++;
+      String name = fetchNames.get(i);
+      ssb.add(getRepositoryConnectionKey(name));
+      objectDescriptions[i] = new RepositoryConnectionDescription(name,new StringSet(ssb));
     }
 
     RepositoryConnectionExecutor exec = new RepositoryConnectionExecutor(this,objectDescriptions);
     cacheManager.findObjectsAndExecute(objectDescriptions,null,exec,getTransactionID());
-    return exec.getResults();
+    IRepositoryConnection[] results = exec.getResults();
+    for (IRepositoryConnection result : results)
+    {
+      rval[outputIndex++] = result;
+    }
+    return outputIndex;
   }
 
   /** Create a new repository connection object.