You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/11/27 15:50:33 UTC
svn commit: r1642164 - in /manifoldcf/branches/dev_1x: ./ framework/
framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/
framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/
framework/pull-agen...
Author: kwright
Date: Thu Nov 27 14:50:32 2014
New Revision: 1642164
URL: http://svn.apache.org/r1642164
Log:
Pull up refactor for CONNECTORS-1116
Modified:
manifoldcf/branches/dev_1x/ (props changed)
manifoldcf/branches/dev_1x/framework/ (props changed)
manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk:r1642163
Propchange: manifoldcf/branches/dev_1x/framework/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk/framework:r1642163
Modified: manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java?rev=1642164&r1=1642163&r2=1642164&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java (original)
+++ manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java Thu Nov 27 14:50:32 2014
@@ -241,31 +241,27 @@ public class OutputConnectionManager ext
IOutputConnection[] rval = new IOutputConnection[names.length];
if (names.length == 0)
return rval;
- List<String> fetchNames = new ArrayList<String>();
+ int inputIndex = 0;
int outputIndex = 0;
- for (String name : names)
+ while (names.length - inputIndex > FETCH_MAX)
{
- if (fetchNames.size() == FETCH_MAX)
- {
- outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
- fetchNames.clear();
- }
- fetchNames.add(name);
+ outputIndex = loadMultipleInternal(rval,outputIndex,names,inputIndex,FETCH_MAX);
+ inputIndex += FETCH_MAX;
}
- loadMultipleInternal(rval,outputIndex,fetchNames);
+ loadMultipleInternal(rval,outputIndex,names,inputIndex,names.length-inputIndex);
return rval;
}
- protected int loadMultipleInternal(IOutputConnection[] rval, int outputIndex, List<String> fetchNames)
+ protected int loadMultipleInternal(IOutputConnection[] rval, int outputIndex, String[] fetchNames, int inputIndex, int length)
throws ManifoldCFException
{
// Build description objects
- OutputConnectionDescription[] objectDescriptions = new OutputConnectionDescription[fetchNames.size()];
+ OutputConnectionDescription[] objectDescriptions = new OutputConnectionDescription[length];
StringSetBuffer ssb = new StringSetBuffer();
- for (int i = 0; i < fetchNames.size(); i++)
+ for (int i = 0; i < length; i++)
{
ssb.clear();
- String name = fetchNames.get(i);
+ String name = fetchNames[inputIndex + i];
ssb.add(getOutputConnectionKey(name));
objectDescriptions[i] = new OutputConnectionDescription(name,new StringSet(ssb));
}
Modified: manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java?rev=1642164&r1=1642163&r2=1642164&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java (original)
+++ manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java Thu Nov 27 14:50:32 2014
@@ -241,32 +241,27 @@ public class TransformationConnectionMan
ITransformationConnection[] rval = new ITransformationConnection[names.length];
if (names.length == 0)
return rval;
- List<String> fetchNames = new ArrayList<String>();
+ int inputIndex = 0;
int outputIndex = 0;
- for (String name : names)
+ while (names.length - inputIndex > FETCH_MAX)
{
- if (fetchNames.size() == FETCH_MAX)
- {
- outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
- fetchNames.clear();
- }
- fetchNames.add(name);
+ outputIndex = loadMultipleInternal(rval,outputIndex,names,inputIndex,FETCH_MAX);
+ inputIndex += FETCH_MAX;
}
- loadMultipleInternal(rval,outputIndex,fetchNames);
+ loadMultipleInternal(rval,outputIndex,names,inputIndex,names.length-inputIndex);
return rval;
}
- protected int loadMultipleInternal(ITransformationConnection[] rval, int outputIndex, List<String> fetchNames)
+ protected int loadMultipleInternal(ITransformationConnection[] rval, int outputIndex, String[] fetchNames, int inputIndex, int length)
throws ManifoldCFException
{
-
// Build description objects
- TransformationConnectionDescription[] objectDescriptions = new TransformationConnectionDescription[fetchNames.size()];
+ TransformationConnectionDescription[] objectDescriptions = new TransformationConnectionDescription[length];
StringSetBuffer ssb = new StringSetBuffer();
- for (int i = 0; i < fetchNames.size(); i++)
+ for (int i = 0; i < length; i++)
{
ssb.clear();
- String name = fetchNames.get(i);
+ String name = fetchNames[inputIndex + i];
ssb.add(getTransformationConnectionKey(name));
objectDescriptions[i] = new TransformationConnectionDescription(name,new StringSet(ssb));
}
Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java?rev=1642164&r1=1642163&r2=1642164&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java Thu Nov 27 14:50:32 2014
@@ -433,31 +433,27 @@ public class AuthorityConnectionManager
IAuthorityConnection[] rval = new IAuthorityConnection[names.length];
if (names.length == 0)
return rval;
- List<String> fetchNames = new ArrayList<String>();
+ int inputIndex = 0;
int outputIndex = 0;
- for (String name : names)
+ while (names.length - inputIndex > FETCH_MAX)
{
- if (fetchNames.size() == FETCH_MAX)
- {
- outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
- fetchNames.clear();
- }
- fetchNames.add(name);
+ outputIndex = loadMultipleInternal(rval,outputIndex,names,inputIndex,FETCH_MAX);
+ inputIndex += FETCH_MAX;
}
- loadMultipleInternal(rval,outputIndex,fetchNames);
+ loadMultipleInternal(rval,outputIndex,names,inputIndex,names.length-inputIndex);
return rval;
}
- protected int loadMultipleInternal(IAuthorityConnection[] rval, int outputIndex, List<String> fetchNames)
+ protected int loadMultipleInternal(IAuthorityConnection[] rval, int outputIndex, String[] fetchNames, int inputIndex, int length)
throws ManifoldCFException
{
// Build description objects
- AuthorityConnectionDescription[] objectDescriptions = new AuthorityConnectionDescription[fetchNames.size()];
+ AuthorityConnectionDescription[] objectDescriptions = new AuthorityConnectionDescription[length];
StringSetBuffer ssb = new StringSetBuffer();
- for (int i = 0; i < fetchNames.size(); i++)
+ for (int i = 0; i < length; i++)
{
ssb.clear();
- String name = fetchNames.get(i);
+ String name = fetchNames[inputIndex + i];
ssb.add(getAuthorityConnectionKey(name));
objectDescriptions[i] = new AuthorityConnectionDescription(name,new StringSet(ssb));
}
Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java?rev=1642164&r1=1642163&r2=1642164&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java Thu Nov 27 14:50:32 2014
@@ -345,31 +345,27 @@ public class MappingConnectionManager ex
IMappingConnection[] rval = new IMappingConnection[names.length];
if (names.length == 0)
return rval;
- List<String> fetchNames = new ArrayList<String>();
+ int inputIndex = 0;
int outputIndex = 0;
- for (String name : names)
+ while (names.length - inputIndex > FETCH_MAX)
{
- if (fetchNames.size() == FETCH_MAX)
- {
- outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
- fetchNames.clear();
- }
- fetchNames.add(name);
+ outputIndex = loadMultipleInternal(rval,outputIndex,names,inputIndex,FETCH_MAX);
+ inputIndex += FETCH_MAX;
}
- loadMultipleInternal(rval,outputIndex,fetchNames);
+ loadMultipleInternal(rval,outputIndex,names,inputIndex,names.length-inputIndex);
return rval;
}
- protected int loadMultipleInternal(IMappingConnection[] rval, int outputIndex, List<String> fetchNames)
+ protected int loadMultipleInternal(IMappingConnection[] rval, int outputIndex, String[] fetchNames, int inputIndex, int length)
throws ManifoldCFException
{
// Build description objects
- MappingConnectionDescription[] objectDescriptions = new MappingConnectionDescription[fetchNames.size()];
+ MappingConnectionDescription[] objectDescriptions = new MappingConnectionDescription[length];
StringSetBuffer ssb = new StringSetBuffer();
- for (int i = 0; i < fetchNames.size(); i++)
+ for (int i = 0; i < length; i++)
{
ssb.clear();
- String name = fetchNames.get(i);
+ String name = fetchNames[inputIndex + i];
ssb.add(getMappingConnectionKey(name));
objectDescriptions[i] = new MappingConnectionDescription(name,new StringSet(ssb));
}
Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java?rev=1642164&r1=1642163&r2=1642164&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java Thu Nov 27 14:50:32 2014
@@ -738,7 +738,7 @@ public class Jobs extends org.apache.man
}
// Only fetch 200 jobs at a time, for resource reasons
- protected final int MAX_FETCH = 200;
+ protected final int FETCH_MAX = 200;
/** Get a list of all jobs which are not in the process of being deleted already.
*@return the array of all jobs.
@@ -916,34 +916,27 @@ public class Jobs extends org.apache.man
IJobDescription[] rval = new IJobDescription[ids.length];
if (ids.length == 0)
return rval;
-
- List<Long> idsToDo = new ArrayList<Long>();
- List<Boolean> readOnliesToDo = new ArrayList<Boolean>();
+
+ int inputIndex = 0;
int outputIndex = 0;
- for (int i = 0; i < ids.length; i++)
+ while (ids.length - inputIndex > FETCH_MAX)
{
- if (idsToDo.size() == MAX_FETCH)
- {
- outputIndex = loadMultipleInternal(rval,outputIndex,idsToDo,readOnliesToDo);
- idsToDo.clear();
- readOnliesToDo.clear();
- }
- idsToDo.add(ids[i]);
- readOnliesToDo.add(readOnlies[i]);
+ outputIndex = loadMultipleInternal(rval,outputIndex,ids,readOnlies,inputIndex,FETCH_MAX);
+ inputIndex += FETCH_MAX;
}
- loadMultipleInternal(rval,outputIndex,idsToDo,readOnliesToDo);
+ loadMultipleInternal(rval,outputIndex,ids,readOnlies,inputIndex,ids.length-inputIndex);
return rval;
}
- protected int loadMultipleInternal(IJobDescription[] rval, int outputIndex, List<Long> ids, List<Boolean> readOnlies)
+ protected int loadMultipleInternal(IJobDescription[] rval, int outputIndex, Long[] ids, boolean[] readOnlies, int inputIndex, int length)
throws ManifoldCFException
{
// Build description objects
- JobObjectDescription[] objectDescriptions = new JobObjectDescription[ids.size()];
+ JobObjectDescription[] objectDescriptions = new JobObjectDescription[length];
StringSetBuffer ssb = new StringSetBuffer();
- for (int i = 0; i < ids.size(); i++)
+ for (int i = 0; i < length; i++)
{
- Long id = ids.get(i);
+ Long id = ids[inputIndex + i];
ssb.clear();
ssb.add(getJobIDKey(id));
objectDescriptions[i] = new JobObjectDescription(id,new StringSet(ssb));
@@ -951,7 +944,7 @@ public class Jobs extends org.apache.man
JobObjectExecutor exec = new JobObjectExecutor(this,objectDescriptions);
cacheManager.findObjectsAndExecute(objectDescriptions,null,exec,getTransactionID());
- IJobDescription[] results = exec.getResults(readOnlies);
+ IJobDescription[] results = exec.getResults(readOnlies,inputIndex);
for (IJobDescription result : results)
{
rval[outputIndex++] = result;
@@ -3714,14 +3707,14 @@ public class Jobs extends org.apache.man
/** Get the result.
*@return the looked-up or read cached instances.
*/
- public JobDescription[] getResults(List<Boolean> readOnlies)
+ public JobDescription[] getResults(boolean[] readOnlies, int inputIndex)
{
JobDescription[] rval = new JobDescription[returnValues.length];
for (int i = 0; i < rval.length; i++)
{
JobDescription jd = returnValues[i];
if (jd != null)
- rval[i] = jd.duplicate(readOnlies.get(i));
+ rval[i] = jd.duplicate(readOnlies[inputIndex + i]);
else
rval[i] = null;
}
Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java?rev=1642164&r1=1642163&r2=1642164&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java Thu Nov 27 14:50:32 2014
@@ -359,31 +359,27 @@ public class RepositoryConnectionManager
IRepositoryConnection[] rval = new IRepositoryConnection[names.length];
if (names.length == 0)
return rval;
- List<String> fetchNames = new ArrayList<String>();
+ int inputIndex = 0;
int outputIndex = 0;
- for (String name : names)
+ while (names.length - inputIndex > FETCH_MAX)
{
- if (fetchNames.size() == FETCH_MAX)
- {
- outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
- fetchNames.clear();
- }
- fetchNames.add(name);
+ outputIndex = loadMultipleInternal(rval,outputIndex,names,inputIndex,FETCH_MAX);
+ inputIndex += FETCH_MAX;
}
- loadMultipleInternal(rval,outputIndex,fetchNames);
+ loadMultipleInternal(rval,outputIndex,names,inputIndex,names.length-inputIndex);
return rval;
}
- protected int loadMultipleInternal(IRepositoryConnection[] rval, int outputIndex, List<String> fetchNames)
+ protected int loadMultipleInternal(IRepositoryConnection[] rval, int outputIndex, String[] fetchNames, int inputIndex, int length)
throws ManifoldCFException
{
// Build description objects
- RepositoryConnectionDescription[] objectDescriptions = new RepositoryConnectionDescription[fetchNames.size()];
+ RepositoryConnectionDescription[] objectDescriptions = new RepositoryConnectionDescription[length];
StringSetBuffer ssb = new StringSetBuffer();
- for (int i = 0; i < fetchNames.size(); i++)
+ for (int i = 0; i < length; i++)
{
ssb.clear();
- String name = fetchNames.get(i);
+ String name = fetchNames[inputIndex + i];
ssb.add(getRepositoryConnectionKey(name));
objectDescriptions[i] = new RepositoryConnectionDescription(name,new StringSet(ssb));
}