You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/12/07 16:04:19 UTC
svn commit: r1548922 - in /manifoldcf/branches/CONNECTORS-781:
connectors/activedirectory/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/activedirectory/
connectors/alfresco/connector/src/main/java/org/apache/manifoldcf/crawler/c...
Author: kwright
Date: Sat Dec 7 15:04:19 2013
New Revision: 1548922
URL: http://svn.apache.org/r1548922
Log:
Implement distributive connection pooling. Still not completely debugged.
Modified:
manifoldcf/branches/CONNECTORS-781/connectors/activedirectory/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/activedirectory/ActiveDirectoryAuthority.java
manifoldcf/branches/CONNECTORS-781/connectors/alfresco/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfresco/AlfrescoRepositoryConnector.java
manifoldcf/branches/CONNECTORS-781/connectors/cmis/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/cmis/CmisRepositoryConnector.java
manifoldcf/branches/CONNECTORS-781/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/authorities/DCTM/AuthorityConnector.java
manifoldcf/branches/CONNECTORS-781/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/DCTM/DCTM.java
manifoldcf/branches/CONNECTORS-781/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java
manifoldcf/branches/CONNECTORS-781/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
manifoldcf/branches/CONNECTORS-781/connectors/filenet/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filenet/FilenetConnector.java
manifoldcf/branches/CONNECTORS-781/connectors/generic/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/generic/GenericAuthority.java
manifoldcf/branches/CONNECTORS-781/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
manifoldcf/branches/CONNECTORS-781/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
manifoldcf/branches/CONNECTORS-781/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
manifoldcf/branches/CONNECTORS-781/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraAuthorityConnector.java
manifoldcf/branches/CONNECTORS-781/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java
manifoldcf/branches/CONNECTORS-781/connectors/ldap/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/ldap/LDAPAuthority.java
manifoldcf/branches/CONNECTORS-781/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkAuthority.java
manifoldcf/branches/CONNECTORS-781/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
manifoldcf/branches/CONNECTORS-781/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointADAuthority.java
manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java
manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
manifoldcf/branches/CONNECTORS-781/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connector/BaseConnector.java
manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connectorpool/ConnectorPool.java
manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IConnector.java
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java
Modified: manifoldcf/branches/CONNECTORS-781/connectors/activedirectory/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/activedirectory/ActiveDirectoryAuthority.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/activedirectory/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/activedirectory/ActiveDirectoryAuthority.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/activedirectory/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/activedirectory/ActiveDirectoryAuthority.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/activedirectory/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/activedirectory/ActiveDirectoryAuthority.java Sat Dec 7 15:04:19 2013
@@ -214,7 +214,21 @@ public class ActiveDirectoryAuthority ex
super.poll();
}
-
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ for (Map.Entry<String,DCSessionInfo> sessionEntry : sessionInfo.entrySet())
+ {
+ if (sessionEntry.getValue().isOpen())
+ return true;
+ }
+ return false;
+ }
+
/** Close the connection. Call this before discarding the repository connector.
*/
@Override
@@ -917,6 +931,11 @@ public class ActiveDirectoryAuthority ex
closeConnection();
}
+ /** Check if open */
+ protected boolean isOpen()
+ {
+ return ctx != null;
+ }
}
/** Class describing a domain suffix and corresponding domain controller name rule.
Modified: manifoldcf/branches/CONNECTORS-781/connectors/alfresco/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfresco/AlfrescoRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/alfresco/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfresco/AlfrescoRepositoryConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/alfresco/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfresco/AlfrescoRepositoryConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/alfresco/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfresco/AlfrescoRepositoryConnector.java Sat Dec 7 15:04:19 2013
@@ -377,6 +377,16 @@ public class AlfrescoRepositoryConnector
}
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return session != null;
+ }
+
/** Queue "seed" documents. Seed documents are the starting places for crawling activity. Documents
* are seeded when this method calls appropriate methods in the passed in ISeedingActivity object.
*
Modified: manifoldcf/branches/CONNECTORS-781/connectors/cmis/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/cmis/CmisRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/cmis/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/cmis/CmisRepositoryConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/cmis/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/cmis/CmisRepositoryConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/cmis/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/cmis/CmisRepositoryConnector.java Sat Dec 7 15:04:19 2013
@@ -603,6 +603,16 @@ public class CmisRepositoryConnector ext
}
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return session != null;
+ }
+
/** Queue "seed" documents. Seed documents are the starting places for crawling activity. Documents
* are seeded when this method calls appropriate methods in the passed in ISeedingActivity object.
*
Modified: manifoldcf/branches/CONNECTORS-781/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/authorities/DCTM/AuthorityConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/authorities/DCTM/AuthorityConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/authorities/DCTM/AuthorityConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/authorities/DCTM/AuthorityConnector.java Sat Dec 7 15:04:19 2013
@@ -947,6 +947,16 @@ public class AuthorityConnector extends
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return session != null;
+ }
+
/** Disconnect from Documentum.
*/
@Override
Modified: manifoldcf/branches/CONNECTORS-781/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/DCTM/DCTM.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/DCTM/DCTM.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/DCTM/DCTM.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/documentum/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/DCTM/DCTM.java Sat Dec 7 15:04:19 2013
@@ -653,6 +653,16 @@ public class DCTM extends org.apache.man
releaseCheck();
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return session != null;
+ }
+
/** Disconnect from Documentum.
*/
@Override
Modified: manifoldcf/branches/CONNECTORS-781/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java Sat Dec 7 15:04:19 2013
@@ -320,6 +320,16 @@ public class DropboxRepositoryConnector
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return session != null;
+ }
+
/**
* Get the maximum number of documents to amalgamate together into one
* batch, for this connector.
Modified: manifoldcf/branches/CONNECTORS-781/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java Sat Dec 7 15:04:19 2013
@@ -168,6 +168,16 @@ public class ElasticSearchConnector exte
expirationTime = -1L;
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return connectionManager != null;
+ }
+
@Override
public void disconnect()
throws ManifoldCFException
Modified: manifoldcf/branches/CONNECTORS-781/connectors/filenet/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filenet/FilenetConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/filenet/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filenet/FilenetConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/filenet/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filenet/FilenetConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/filenet/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filenet/FilenetConnector.java Sat Dec 7 15:04:19 2013
@@ -473,6 +473,16 @@ public class FilenetConnector extends or
releaseCheck();
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return session != null;
+ }
+
/** Disconnect from Filenet.
*/
@Override
Modified: manifoldcf/branches/CONNECTORS-781/connectors/generic/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/generic/GenericAuthority.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/generic/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/generic/GenericAuthority.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/generic/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/generic/GenericAuthority.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/generic/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/generic/GenericAuthority.java Sat Dec 7 15:04:19 2013
@@ -188,6 +188,16 @@ public class GenericAuthority extends or
super.poll();
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return client != null;
+ }
+
/**
* Check connection for sanity.
*/
Modified: manifoldcf/branches/CONNECTORS-781/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java Sat Dec 7 15:04:19 2013
@@ -384,6 +384,16 @@ public class GoogleDriveRepositoryConnec
}
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return session != null;
+ }
+
/**
* Get the maximum number of documents to amalgamate together into one
* batch, for this connector.
Modified: manifoldcf/branches/CONNECTORS-781/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java Sat Dec 7 15:04:19 2013
@@ -118,6 +118,16 @@ public class HDFSOutputConnector extends
user = configParams.getParameter(ParameterEnum.user.name());
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return session != null;
+ }
+
/** Close the connection. Call this before discarding the connection.
*/
@Override
Modified: manifoldcf/branches/CONNECTORS-781/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java Sat Dec 7 15:04:19 2013
@@ -220,6 +220,16 @@ public class HDFSRepositoryConnector ext
}
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return session != null;
+ }
+
/**
* @throws ManifoldCFException
*/
Modified: manifoldcf/branches/CONNECTORS-781/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraAuthorityConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraAuthorityConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraAuthorityConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraAuthorityConnector.java Sat Dec 7 15:04:19 2013
@@ -223,6 +223,16 @@ public class JiraAuthorityConnector exte
return session;
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return session != null;
+ }
+
@Override
public void poll() throws ManifoldCFException {
if (lastSessionFetch == -1L) {
Modified: manifoldcf/branches/CONNECTORS-781/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java Sat Dec 7 15:04:19 2013
@@ -298,6 +298,16 @@ public class JiraRepositoryConnector ext
return session;
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return session != null;
+ }
+
@Override
public void poll() throws ManifoldCFException {
if (lastSessionFetch == -1L) {
Modified: manifoldcf/branches/CONNECTORS-781/connectors/ldap/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/ldap/LDAPAuthority.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/ldap/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/ldap/LDAPAuthority.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/ldap/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/ldap/LDAPAuthority.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/ldap/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/ldap/LDAPAuthority.java Sat Dec 7 15:04:19 2013
@@ -224,6 +224,16 @@ public class LDAPAuthority extends org.a
return super.check();
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return session != null;
+ }
+
/**
* Poll. The connection should be closed if it has been idle for too long.
*/
Modified: manifoldcf/branches/CONNECTORS-781/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkAuthority.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkAuthority.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkAuthority.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkAuthority.java Sat Dec 7 15:04:19 2013
@@ -317,6 +317,16 @@ public class LivelinkAuthority extends o
}
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return hasConnected;
+ }
+
/** Close the connection. Call this before discarding the repository connector.
*/
@Override
Modified: manifoldcf/branches/CONNECTORS-781/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java Sat Dec 7 15:04:19 2013
@@ -704,6 +704,16 @@ public class LivelinkConnector extends o
}
}
}
+
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return hasConnected;
+ }
/** Close the connection. Call this before discarding the repository connector.
*/
Modified: manifoldcf/branches/CONNECTORS-781/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java Sat Dec 7 15:04:19 2013
@@ -199,6 +199,16 @@ public class OpenSearchServerConnector e
}
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return connectionManager != null;
+ }
+
@Override
public String[] getActivitiesList() {
return OPENSEARCHSERVER_ACTIVITIES;
Modified: manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointADAuthority.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointADAuthority.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointADAuthority.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointADAuthority.java Sat Dec 7 15:04:19 2013
@@ -220,6 +220,21 @@ public class SharePointADAuthority exten
super.poll();
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ for (Map.Entry<String,DCSessionInfo> sessionEntry : sessionInfo.entrySet())
+ {
+ if (sessionEntry.getValue().isOpen())
+ return true;
+ }
+ return false;
+ }
+
/** Close the connection. Call this before discarding the repository connector.
*/
@Override
@@ -942,6 +957,12 @@ public class SharePointADAuthority exten
closeConnection();
}
+ /** Check if open */
+ protected boolean isOpen()
+ {
+ return ctx != null;
+ }
+
}
/** Class describing a domain suffix and corresponding domain controller name rule.
Modified: manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java Sat Dec 7 15:04:19 2013
@@ -243,7 +243,17 @@ public class SharePointAuthority extends
connectionManager.closeIdleConnections(60000L,TimeUnit.MILLISECONDS);
super.poll();
}
-
+
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return connectionManager != null;
+ }
+
/** Close the connection. Call this before discarding the repository connector.
*/
@Override
Modified: manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java Sat Dec 7 15:04:19 2013
@@ -423,6 +423,16 @@ public class SharePointRepository extend
connectionManager.closeIdleConnections(60000L,TimeUnit.MILLISECONDS);
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return connectionManager != null;
+ }
+
/** Request arbitrary connector information.
* This method is called directly from the API in order to allow API users to perform any one of several connector-specific
* queries.
Modified: manifoldcf/branches/CONNECTORS-781/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java Sat Dec 7 15:04:19 2013
@@ -110,6 +110,16 @@ public class SolrConnector extends org.a
}
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return poster != null;
+ }
+
/** Close the connection. Call this before discarding the connection.
*/
@Override
Modified: manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connector/BaseConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connector/BaseConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connector/BaseConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connector/BaseConnector.java Sat Dec 7 15:04:19 2013
@@ -39,6 +39,7 @@ public abstract class BaseConnector impl
* It is called when the connector is registered.
*@param threadContext is the current thread context.
*/
+ @Override
public void install(IThreadContext threadContext)
throws ManifoldCFException
{
@@ -50,6 +51,7 @@ public abstract class BaseConnector impl
* It is called when the connector is deregistered.
*@param threadContext is the current thread context.
*/
+ @Override
public void deinstall(IThreadContext threadContext)
throws ManifoldCFException
{
@@ -59,6 +61,7 @@ public abstract class BaseConnector impl
/** Connect. The configuration parameters are included.
*@param configParams are the configuration parameters for this connection.
*/
+ @Override
public void connect(ConfigParams configParams)
{
params = configParams;
@@ -70,6 +73,7 @@ public abstract class BaseConnector impl
/** Test the connection. Returns a string describing the connection integrity.
*@return the connection's status as a displayable string.
*/
+ @Override
public String check()
throws ManifoldCFException
{
@@ -80,14 +84,27 @@ public abstract class BaseConnector impl
/** This method is periodically called for all connectors that are connected but not
* in active use.
*/
+ @Override
public void poll()
throws ManifoldCFException
{
// Base version does nothing
}
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ // Consider it connected.
+ return true;
+ }
+
/** Close the connection. Call this before discarding the repository connector.
*/
+ @Override
public void disconnect()
throws ManifoldCFException
{
@@ -97,6 +114,7 @@ public abstract class BaseConnector impl
/** Clear out any state information specific to a given thread.
* This method is called when this object is returned to the connection pool.
*/
+ @Override
public void clearThreadContext()
{
currentContext = null;
@@ -105,6 +123,7 @@ public abstract class BaseConnector impl
/** Attach to a new thread.
*@param threadContext is the new thread context.
*/
+ @Override
public void setThreadContext(IThreadContext threadContext)
throws ManifoldCFException
{
@@ -114,6 +133,7 @@ public abstract class BaseConnector impl
/** Get configuration information.
*@return the configuration information for this class.
*/
+ @Override
public ConfigParams getConfiguration()
{
return params;
@@ -128,6 +148,7 @@ public abstract class BaseConnector impl
*@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
*@param tabsArray is an array of tab names. Add to this array any tab names that are specific to the connector.
*/
+ @Override
public void outputConfigurationHeader(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters, List<String> tabsArray)
throws ManifoldCFException, IOException
{
@@ -169,6 +190,7 @@ public abstract class BaseConnector impl
*@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
*@param tabName is the current tab name.
*/
+ @Override
public void outputConfigurationBody(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters, String tabName)
throws ManifoldCFException, IOException
{
@@ -199,6 +221,7 @@ public abstract class BaseConnector impl
*@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
*@return null if all is well, or a string error message if there is an error that should prevent saving of the connection (and cause a redirection to an error page).
*/
+ @Override
public String processConfigurationPost(IThreadContext threadContext, IPostParameters variableContext, Locale locale, ConfigParams parameters)
throws ManifoldCFException
{
@@ -228,6 +251,7 @@ public abstract class BaseConnector impl
*@param locale is the locale that the output should use.
*@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
*/
+ @Override
public void viewConfiguration(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters)
throws ManifoldCFException, IOException
{
Modified: manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connectorpool/ConnectorPool.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connectorpool/ConnectorPool.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connectorpool/ConnectorPool.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/connectorpool/ConnectorPool.java Sat Dec 7 15:04:19 2013
@@ -47,6 +47,44 @@ public abstract class ConnectorPool<T ex
// "data type" name - only in the local pool will we pay any attention to config info and class name, and flush those handles
// that get returned that have the wrong info attached.
+ // Gating whether a connector instance should be created occurs when someone tries to grab a connector instance.
+ // When it comes time to get a handle in the local pool, there are two situations:
+ // (1) We know that the handle is available locally.
+ // (2) The handle is not available locally, but may be available globally.
+ // In case (2), we wait until the allocated global handle count is less than the maximum, and only then do we grab a handle
+ // (incrementing the service counter when we do that). This requires polling - but we will poll relatively infrequently.
+
+ // Determining how many connections THIS pool instance should hold on to locally is done at the time a connector
+ // instance is freed. A decision is made whether to discard the instance (or another that we're keeping around),
+ // or keep it. The main factors that go into that decision are the following:
+ // - Are we in excess of the local fixed quota?
+ // - Are we in excess of the local transient quota?
+ // The quotas are determined at polling time (?). The transient quota is calculated by each instance, and uses the
+ // following information: (a) how many registered services are there; (b) how many total handles are allowed; (c)
+ // how many handles are actually connected at the time for each of services. The goal is to apportion handles to the
+ // services that are actually using them; this can change dynamically over time.
+
+ // Two numbers each service posts: "in-use" and "target". At no time does a service *ever* post either a "target"
+ // that, together with all other active service targets, is in excess of the max. Also, at no time a service post
+ // a target that, when added to the other "in-use" values, exceeds the max. If the "in-use" values everywhere else
+ // already equal or exceed the max, then the target will be zero.
+ // The target quota is calculated as follows:
+ // (1) Target is summed, excluding ours. This is GlobalTarget.
+ // (2) In-use is summed, excluding ours. This is GlobalInUse.
+ // (3) Our MaximumTarget is computed, which is Maximum - GlobalTarget or Maximum - GlobalInUse, whichever is
+ // smaller, but never less than zero.
+ // (4) Our FairTarget is computed. The FairTarget divides the Maximum by the number of services, and adds
+ // 1 randomly based on the remainder.
+ // (5) We compute OptimalTarget as follows: We start with current local target. If current local target
+ // exceeds current local in-use count, we adjust OptimalTarget downward by one. Otherwise we increase it
+ // by one.
+ // (6) Finally, we compute Target by taking the minimum of MaximumTarget, FairTarget, and OptimalTarget.
+
+ /** Target data name */
+ protected final String targetDataType = "target";
+
+ /** In-use data name */
+ protected final String inUseDataType = "in-use";
/** Service type prefix */
protected final String serviceTypePrefix;
@@ -54,6 +92,9 @@ public abstract class ConnectorPool<T ex
/** Pool hash table. Keyed by connection name; value is Pool */
protected final Map<String,Pool> poolHash = new HashMap<String,Pool>();
+ /** Random number */
+ protected final static Random randomNumberGenerator = new Random();
+
protected ConnectorPool(String serviceTypePrefix)
{
this.serviceTypePrefix = serviceTypePrefix;
@@ -367,19 +408,29 @@ public abstract class ConnectorPool<T ex
*/
protected class Pool
{
+ /** Whether the pool has been shut down or not */
+ protected boolean isAlive = true;
+ /** The global maximum for this pool */
+ protected int globalMax;
+ /** Service type name */
protected final String serviceTypeName;
+ /** The (anonymous) service name */
protected final String serviceName;
+ /** Place where we keep unused connector instances */
protected final List<T> stack = new ArrayList<T>();
- protected int globalMax;
- protected int numFree;
-
+ /** The number of local instances we can currently pass out to requesting threads. Initially zero until pool is apportioned */
+ protected int numFree = 0;
+ /** The number of instances we are allowed to hand out locally, at this time */
+ protected int localMax = 0;
+ /** The number of instances that are actually connected and in use, as of the last poll */
+ protected int localInUse = 0;
+
/** Constructor
*/
public Pool(IThreadContext threadContext, int maxCount, String connectionName)
throws ManifoldCFException
{
- this.globalMax = globalMax;
- this.numFree = maxCount;
+ this.globalMax = maxCount;
this.serviceTypeName = buildServiceTypeName(connectionName);
// Now, register and activate service anonymously, and record the service name we get.
ILockManager lockManager = LockManagerFactory.make(threadContext);
@@ -392,15 +443,12 @@ public abstract class ConnectorPool<T ex
public synchronized void updateMaximumPoolSize(IThreadContext threadContext, int maxPoolSize)
throws ManifoldCFException
{
- // Compute the number of instances in use locally
- int localInUse = globalMax - numFree;
+ // This updates the maximum global size that the pool uses.
globalMax = maxPoolSize;
- // numFree may turn out to be negative here!! That's okay; we'll just free released connectors
- // until we enter positive territory again.
- numFree = globalMax - localInUse;
- notifyAll();
+ // We do nothing else at this time; we rely on polling to reapportion the pool.
}
+
/** Grab a connector.
* If none exists, construct it using the information in the pool key.
*@return the connector, or null if no connector could be connected.
@@ -484,7 +532,20 @@ public abstract class ConnectorPool<T ex
// Simplifying: excess is when stack.size() > numFree.
while (stack.size() > 0 && stack.size() > numFree)
{
- T rc = stack.remove(stack.size()-1);
+ // Try to find a connector instance that is not actually connected.
+ // These are likely to be at the front of the queue, since those are the
+ // oldest.
+ int j;
+ for (j = 0; j < stack.size(); j++)
+ {
+ if (!stack.get(j).isConnected())
+ break;
+ }
+ T rc;
+ if (j == stack.size())
+ rc = stack.remove(stack.size()-1);
+ else
+ rc = stack.remove(j);
rc.setThreadContext(threadContext);
try
{
@@ -504,23 +565,103 @@ public abstract class ConnectorPool<T ex
public synchronized void pollAll(IThreadContext threadContext)
throws ManifoldCFException
{
- int i = 0;
- while (i < stack.size())
+ // The meat of the cross-cluster apportionment algorithm goes here!
+ // Two global numbers each service posts: "in-use" and "target". At no time does a service *ever* post either a "target"
+ // that, together with all other active service targets, is in excess of the max. Also, at no time a service post
+ // a target that, when added to the other "in-use" values, exceeds the max. If the "in-use" values everywhere else
+ // already equal or exceed the max, then the target will be zero.
+ // The target quota is calculated as follows:
+ // (1) Target is summed, excluding ours. This is GlobalTarget.
+ // (2) In-use is summed, excluding ours. This is GlobalInUse.
+ // (3) Our MaximumTarget is computed, which is Maximum - GlobalTarget or Maximum - GlobalInUse, whichever is
+ // smaller, but never less than zero.
+ // (4) Our FairTarget is computed. The FairTarget divides the Maximum by the number of services, and adds
+ // 1 randomly based on the remainder.
+ // (5) We compute OptimalTarget as follows: We start with current local target. If current local target
+ // exceeds current local in-use count, we adjust OptimalTarget downward by one. Otherwise we increase it
+ // by one.
+ // (6) Finally, we compute Target by taking the minimum of MaximumTarget, FairTarget, and OptimalTarget.
+
+ System.out.println("In pollAll for "+serviceTypeName+" "+serviceName);
+ if (!isAlive)
+ return;
+
+ System.out.println("Is alive in pollAll for "+serviceTypeName+" "+serviceName);
+
+ ILockManager lockManager = LockManagerFactory.make(threadContext);
+
+ int numServices = lockManager.countActiveServices(serviceTypeName);
+ if (numServices == 0)
+ // Should never happen, but if it does just give up.
+ return;
+
+ System.out.println("Has active services in pollAll for "+serviceTypeName+" "+serviceName);
+
+ // Compute MaximumTarget
+ int globalTarget = computeGlobalSum(lockManager, targetDataType);
+ int globalInUse = computeGlobalSum(lockManager, inUseDataType);
+ int maximumTarget = globalMax - globalTarget;
+ if (maximumTarget > globalMax - globalInUse)
+ maximumTarget = globalMax - globalInUse;
+ if (maximumTarget < 0)
+ maximumTarget = 0;
+
+ // Compute FairTarget
+ int fairTarget = globalMax / numServices;
+ int remainder = globalMax % numServices;
+ // Randomly choose whether we get an addition to the FairTarget
+ if (randomNumberGenerator.nextInt(numServices) <= remainder)
+ fairTarget++;
+
+ // Compute OptimalTarget (and poll connectors while we are at it)
+ int localInUse = localMax - numFree; // These are the connectors that have been handed out
+ for (T rc : stack)
{
- T rc = stack.get(i++);
// Notify
rc.setThreadContext(threadContext);
try
{
rc.poll();
+ if (rc.isConnected())
+ localInUse++; // Count every pooled connector that is still connected
}
finally
{
rc.clearThreadContext();
}
}
+ int optimalTarget = localMax;
+ if (localMax > localInUse)
+ optimalTarget--;
+ else
+ optimalTarget++;
+
+ // Now compute actual target
+ int target = maximumTarget;
+ if (target > fairTarget)
+ target = fairTarget;
+ if (target > optimalTarget)
+ target = optimalTarget;
+
+ System.out.println("Max target: "+maximumTarget+"; fair target: "+fairTarget+"; optimal target: "+optimalTarget+"; localInUse: "+localInUse);
+
+ // Write these values to the service data variables
+ // NOTE that these can be combined to one, for significant efficiencies MHL
+ lockManager.updateServiceData(serviceTypeName, serviceName, inUseDataType, packInt(localInUse));
+ lockManager.updateServiceData(serviceTypeName, serviceName, targetDataType, packInt(target));
+
+ // Now, update our localMax
+ changeLocalMaxValue(target);
}
+ protected int computeGlobalSum(ILockManager lockManager, String dataType)
+ throws ManifoldCFException
+ {
+ SumClass sumClass = new SumClass(serviceName);
+ lockManager.scanServiceData(serviceTypeName, dataType, sumClass);
+ return sumClass.getSum();
+ }
+
/** Flush unused connectors.
*/
public synchronized void flushUnused(IThreadContext threadContext)
@@ -548,11 +689,79 @@ public abstract class ConnectorPool<T ex
throws ManifoldCFException
{
flushUnused(threadContext);
+ System.out.println("ENDING SERVICE ACTIVITY!! for "+serviceTypeName+" "+serviceName);
+ new Exception("Bad stuff").printStackTrace();
+
// End service activity
+ isAlive = false;
ILockManager lockManager = LockManagerFactory.make(threadContext);
lockManager.endServiceActivity(serviceTypeName, serviceName);
}
+ // Protected methods
+
+ /** Change the localMax value.
+ *@param newLocalMax is the new local max value.
+ */
+ protected synchronized void changeLocalMaxValue(int newLocalMax)
+ {
+ if (newLocalMax == localMax)
+ return;
+ System.out.println("Changing local max for "+serviceTypeName+" "+serviceName+" to "+newLocalMax);
+ // Compute the number of instances in use locally
+ int localInUse = localMax - numFree;
+ localMax = newLocalMax;
+ // numFree may turn out to be negative here!! That's okay; we'll just free released connectors
+ // until we enter positive territory again.
+ numFree = localMax - localInUse;
+ notifyAll();
+ }
+
}
+ protected static class SumClass implements IServiceDataAcceptor
+ {
+ protected final String serviceName;
+ protected int tally = 0;
+
+ public SumClass(String serviceName)
+ {
+ this.serviceName = serviceName;
+ }
+
+ @Override
+ public boolean acceptServiceData(String serviceName, byte[] serviceData)
+ throws ManifoldCFException
+ {
+ if (!serviceName.equals(this.serviceName))
+ tally += unpackInt(serviceData);
+ return false;
+ }
+
+ public int getSum()
+ {
+ return tally;
+ }
+ }
+
+ protected static int unpackInt(byte[] data)
+ {
+ if (data == null || data.length != 4)
+ return 0;
+ return ((int)data[0]) & 0xff +
+ (((int)data[1]) >> 8) & 0xff +
+ (((int)data[2]) >> 16) & 0xff +
+ (((int)data[3]) >> 24) & 0xff;
+ }
+
+ protected static byte[] packInt(int value)
+ {
+ byte[] rval = new byte[4];
+ rval[0] = (byte)(value & 0xff);
+ rval[1] = (byte)((value >> 8) & 0xff);
+ rval[2] = (byte)((value >> 16) & 0xff);
+ rval[3] = (byte)((value >> 24) & 0xff);
+ return rval;
+ }
+
}
Modified: manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IConnector.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IConnector.java Sat Dec 7 15:04:19 2013
@@ -64,6 +64,12 @@ public interface IConnector
public void poll()
throws ManifoldCFException;
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ public boolean isConnected();
+
/** Close the connection. Call this before discarding the repository connector.
*/
public void disconnect()
Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java?rev=1548922&r1=1548921&r2=1548922&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java Sat Dec 7 15:04:19 2013
@@ -66,7 +66,9 @@ public class IdleCleanupThread extends T
try
{
// Do the cleanup
+ System.out.println("Polling...");
repositoryConnectorPool.pollAllConnectors();
+ System.out.println("...done");
cacheManager.expireObjects(System.currentTimeMillis());
// Sleep for the retry interval.