You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/12/06 15:30:54 UTC
svn commit: r1548537 - in /manifoldcf/branches/CONNECTORS-781/framework:
agents/src/main/java/org/apache/manifoldcf/agents/outputconnectorpool/
core/src/main/java/org/apache/manifoldcf/core/connectorpool/
pull-agent/src/main/java/org/apache/manifoldcf/...
Author: kwright
Date: Fri Dec 6 14:30:54 2013
New Revision: 1548537
URL: http://svn.apache.org/r1548537
Log:
Add logic for adjusting pool size depending on changes to connection info
Modified:
manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnectorpool/OutputConnectorPool.java
manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connectorpool/ConnectorPool.java
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authorityconnectorpool/AuthorityConnectorPool.java
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mappingconnectorpool/MappingConnectorPool.java
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repositoryconnectorpool/RepositoryConnectorPool.java
Modified: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnectorpool/OutputConnectorPool.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnectorpool/OutputConnectorPool.java?rev=1548537&r1=1548536&r2=1548537&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnectorpool/OutputConnectorPool.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnectorpool/OutputConnectorPool.java Fri Dec 6 14:30:54 2013
@@ -35,22 +35,6 @@ public class OutputConnectorPool impleme
/** Local connector pool */
protected final static LocalPool localPool = new LocalPool();
- // How global connector allocation works:
- // (1) There is a lock-manager "service" associated with this connector pool. This allows us to clean
- // up after local pools that have died without being released. There's one anonymous service instance per local pool,
- // and thus one service instance per JVM.
- // (2) Each local pool knows how many connector instances of each type (keyed by class name and config info) there
- // are.
- // (3) Each local pool/connector instance type has a local authorization count. This is the amount it's
- // allowed to actually keep. If the pool has more connectors of a type than the local authorization count permits,
- // then every connector release operation will destroy the released connector until the local authorization count
- // is met.
- // (4) Each local pool/connector instance type needs a global variable describing how many CURRENT instances
- // the local pool has allocated. This is a transient value which should automatically go to zero if the service becomes inactive.
- // The lock manager has primitives now that allow data to be set this way. We will use the connection name as the
- // "data type" name - only in the local pool will we pay any attention to config info and class name, and flush those handles
- // that get returned that have the wrong info attached.
-
/** Thread context */
protected final IThreadContext threadContext;
@@ -113,7 +97,7 @@ public class OutputConnectorPool impleme
{
connectionNames[i] = connections[i].getName();
}
- localPool.releaseMultiple(connectionNames, connectors);
+ localPool.releaseMultiple(threadContext, connectionNames, connectors);
}
/** Release an output connector.
@@ -124,7 +108,7 @@ public class OutputConnectorPool impleme
public void release(IOutputConnection connection, IOutputConnector connector)
throws ManifoldCFException
{
- localPool.release(connection.getName(),connector);
+ localPool.release(threadContext,connection.getName(),connector);
}
/** Idle notification for inactive output connector handles.
Modified: manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connectorpool/ConnectorPool.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connectorpool/ConnectorPool.java?rev=1548537&r1=1548536&r2=1548537&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connectorpool/ConnectorPool.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connectorpool/ConnectorPool.java Fri Dec 6 14:30:54 2013
@@ -31,6 +31,23 @@ public abstract class ConnectorPool<T ex
{
public static final String _rcsid = "@(#)$Id$";
+ // How global connector allocation works:
+ // (1) There is a lock-manager "service" associated with this connector pool. This allows us to clean
+ // up after local pools that have died without being released. There's one anonymous service instance per local pool,
+ // and thus one service instance per JVM.
+ // (2) Each local pool knows how many connector instances of each type (keyed by connection name) there
+ // are.
+ // (3) Each local pool/connector instance type has a local authorization count. This is the amount it's
+ // allowed to actually keep. If the pool has more connectors of a type than the local authorization count permits,
+ // then every connector release operation will destroy the released connector until the local authorization count
+ // is met.
+ // (4) Each local pool/connector instance type needs a global variable describing how many CURRENT instances
+ // the local pool has allocated. This is a transient value which should automatically go to zero if the service becomes inactive.
+ // The lock manager has primitives now that allow data to be set this way. We will use the connection name as the
+ // "data type" name - only in the local pool will we pay any attention to config info and class name, and flush those handles
+ // that get returned that have the wrong info attached.
+
+
/** Service type prefix */
protected final String serviceTypePrefix;
@@ -171,7 +188,7 @@ public abstract class ConnectorPool<T ex
index = orderMap.get(orderingKey).intValue();
try
{
- release(connectionName,rval[index]);
+ release(threadContext,connectionName,rval[index]);
}
catch (ManifoldCFException e2)
{
@@ -217,6 +234,10 @@ public abstract class ConnectorPool<T ex
p = new Pool(threadContext, maxPoolSize, connectionName);
poolHash.put(connectionName,p);
}
+ else
+ {
+ p.updateMaximumPoolSize(threadContext, maxPoolSize);
+ }
}
T rval = p.getConnector(threadContext,className,configInfo);
@@ -227,7 +248,7 @@ public abstract class ConnectorPool<T ex
/** Release multiple output connectors.
*/
- public void releaseMultiple(String[] connectionNames, T[] connectors)
+ public void releaseMultiple(IThreadContext threadContext, String[] connectionNames, T[] connectors)
throws ManifoldCFException
{
ManifoldCFException currentException = null;
@@ -237,7 +258,7 @@ public abstract class ConnectorPool<T ex
T c = connectors[i];
try
{
- release(connectionName,c);
+ release(threadContext,connectionName,c);
}
catch (ManifoldCFException e)
{
@@ -253,7 +274,7 @@ public abstract class ConnectorPool<T ex
*@param connectionName is the connection name.
*@param connector is the connector to release.
*/
- public void release(String connectionName, T connector)
+ public void release(IThreadContext threadContext, String connectionName, T connector)
throws ManifoldCFException
{
// If the connector is null, skip the release, because we never really got the connector in the first place.
@@ -267,7 +288,7 @@ public abstract class ConnectorPool<T ex
p = poolHash.get(connectionName);
}
- p.releaseConnector(connector);
+ p.releaseConnector(threadContext, connector);
}
/** Idle notification for inactive output connector handles.
@@ -349,6 +370,7 @@ public abstract class ConnectorPool<T ex
protected final String serviceTypeName;
protected final String serviceName;
protected final List<T> stack = new ArrayList<T>();
+ protected int globalMax;
protected int numFree;
/** Constructor
@@ -356,6 +378,7 @@ public abstract class ConnectorPool<T ex
public Pool(IThreadContext threadContext, int maxCount, String connectionName)
throws ManifoldCFException
{
+ this.globalMax = globalMax;
this.numFree = maxCount;
this.serviceTypeName = buildServiceTypeName(connectionName);
// Now, register and activate service anonymously, and record the service name we get.
@@ -363,6 +386,21 @@ public abstract class ConnectorPool<T ex
this.serviceName = lockManager.registerServiceBeginServiceActivity(serviceTypeName, null, null);
}
+ /** Update the maximum pool size.
+ *@param maxPoolSize is the new global maximum pool size.
+ */
+ public synchronized void updateMaximumPoolSize(IThreadContext threadContext, int maxPoolSize)
+ throws ManifoldCFException
+ {
+ // Compute the number of instances in use locally
+ int localInUse = globalMax - numFree;
+ globalMax = maxPoolSize;
+ // numFree may turn out to be negative here!! That's okay; we'll just free released connectors
+ // until we enter positive territory again.
+ numFree = globalMax - localInUse;
+ notifyAll();
+ }
+
/** Grab a connector.
* If none exists, construct it using the information in the pool key.
*@return the connector, or null if no connector could be connected.
@@ -372,7 +410,7 @@ public abstract class ConnectorPool<T ex
{
// numFree represents the number of available connector instances that have not been given out at this moment.
// So it's the max minus the pool count minus the number in use.
- while (numFree == 0)
+ while (numFree <= 0)
{
try
{
@@ -425,7 +463,7 @@ public abstract class ConnectorPool<T ex
/** Release a connector to the pool.
*@param connector is the connector.
*/
- public synchronized void releaseConnector(T connector)
+ public synchronized void releaseConnector(IThreadContext threadContext, T connector)
throws ManifoldCFException
{
if (connector == null)
@@ -433,9 +471,31 @@ public abstract class ConnectorPool<T ex
// Make sure connector knows it's released
connector.clearThreadContext();
- // Append
+ // Return it to the pool, and note that it is no longer in use.
stack.add(connector);
numFree++;
+ // Determine if we need to free some connectors. If the number
+ // of allocated connectors exceeds the target, we unload some
+ // off the stack.
+ // The question is whether the stack has too many connector instances
+ // on it. Obviously, if it stack.size() > max, it does - but remember
+ // that the number of outstanding connectors is max - numFree.
+ // So, we have an excess if stack.size() > max - (max-numFree).
+ // Simplifying: excess is when stack.size() > numFree.
+ while (stack.size() > 0 && stack.size() > numFree)
+ {
+ T rc = stack.remove(stack.size()-1);
+ rc.setThreadContext(threadContext);
+ try
+ {
+ rc.disconnect();
+ }
+ finally
+ {
+ rc.clearThreadContext();
+ }
+ }
+
notifyAll();
}
Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authorityconnectorpool/AuthorityConnectorPool.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authorityconnectorpool/AuthorityConnectorPool.java?rev=1548537&r1=1548536&r2=1548537&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authorityconnectorpool/AuthorityConnectorPool.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authorityconnectorpool/AuthorityConnectorPool.java Fri Dec 6 14:30:54 2013
@@ -100,7 +100,7 @@ public class AuthorityConnectorPool impl
{
connectionNames[i] = connections[i].getName();
}
- localPool.releaseMultiple(connectionNames, connectors);
+ localPool.releaseMultiple(threadContext, connectionNames, connectors);
}
/** Release an output connector.
@@ -111,7 +111,7 @@ public class AuthorityConnectorPool impl
public void release(IAuthorityConnection connection, IAuthorityConnector connector)
throws ManifoldCFException
{
- localPool.release(connection.getName(), connector);
+ localPool.release(threadContext, connection.getName(), connector);
}
/** Idle notification for inactive authority connector handles.
Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mappingconnectorpool/MappingConnectorPool.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mappingconnectorpool/MappingConnectorPool.java?rev=1548537&r1=1548536&r2=1548537&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mappingconnectorpool/MappingConnectorPool.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mappingconnectorpool/MappingConnectorPool.java Fri Dec 6 14:30:54 2013
@@ -101,7 +101,7 @@ public class MappingConnectorPool implem
{
connectionNames[i] = connections[i].getName();
}
- localPool.releaseMultiple(connectionNames, connectors);
+ localPool.releaseMultiple(threadContext, connectionNames, connectors);
}
/** Release a mapping connector.
@@ -112,7 +112,7 @@ public class MappingConnectorPool implem
public void release(IMappingConnection connection, IMappingConnector connector)
throws ManifoldCFException
{
- localPool.release(connection.getName(), connector);
+ localPool.release(threadContext, connection.getName(), connector);
}
/** Idle notification for inactive mapping connector handles.
Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repositoryconnectorpool/RepositoryConnectorPool.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repositoryconnectorpool/RepositoryConnectorPool.java?rev=1548537&r1=1548536&r2=1548537&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repositoryconnectorpool/RepositoryConnectorPool.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repositoryconnectorpool/RepositoryConnectorPool.java Fri Dec 6 14:30:54 2013
@@ -100,7 +100,7 @@ public class RepositoryConnectorPool imp
{
connectionNames[i] = connections[i].getName();
}
- localPool.releaseMultiple(connectionNames, connectors);
+ localPool.releaseMultiple(threadContext, connectionNames, connectors);
}
/** Release a repository connector.
@@ -111,7 +111,7 @@ public class RepositoryConnectorPool imp
public void release(IRepositoryConnection connection, IRepositoryConnector connector)
throws ManifoldCFException
{
- localPool.release(connection.getName(), connector);
+ localPool.release(threadContext, connection.getName(), connector);
}
/** Idle notification for inactive repository connector handles.