You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/12/06 02:28:02 UTC
svn commit: r1548357 - in /manifoldcf/branches/CONNECTORS-781/framework:
agents/src/main/java/org/apache/manifoldcf/agents/outputconnectorpool/
core/src/main/java/org/apache/manifoldcf/core/interfaces/
core/src/main/java/org/apache/manifoldcf/core/lock...
Author: kwright
Date: Fri Dec 6 01:28:02 2013
New Revision: 1548357
URL: http://svn.apache.org/r1548357
Log:
Add service data scanner
Added:
manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IServiceDataAcceptor.java (with props)
Modified:
manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnectorpool/OutputConnectorPool.java
manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ILockManager.java
manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/BaseLockManager.java
manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/LockManager.java
manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java
Modified: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnectorpool/OutputConnectorPool.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnectorpool/OutputConnectorPool.java?rev=1548357&r1=1548356&r2=1548357&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnectorpool/OutputConnectorPool.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnectorpool/OutputConnectorPool.java Fri Dec 6 01:28:02 2013
@@ -47,13 +47,9 @@ public class OutputConnectorPool impleme
// is met.
// (4) Each local pool/connector instance type needs a global variable describing how many CURRENT instances
// the local pool has allocated. This is a transient value which should automatically go to zero if the service becomes inactive.
- // This is attached to the service as part of the transient service data for the service.
- // Q: How do we separate instance counts based on pool classname/config??
- // A: This seems challenging because (a) multiple transient variables seem to be needed, and (b) the variable names are VERY long
- // (the entire length of the config!!) Alternatively we could use the name of the connection, but then when the connection info changes,
- // we would need to address this somehow in the pool. How??
- // The lock manager will need to be extended in order to provide this functionality - essentially, transient service-associated data.
- // There also needs to be a fast way to sum the data for all active services (but that does not need to be a lock-manager primitive)
+ // The lock manager has primitives now that allow data to be set this way. We will use the connection name as the
+ // "data type" name - only in the local pool will we pay any attention to config info and class name, and flush those handles
+ // that get returned that have the wrong info attached.
/** Thread context */
protected final IThreadContext threadContext;
Modified: manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ILockManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ILockManager.java?rev=1548357&r1=1548356&r2=1548357&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ILockManager.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ILockManager.java Fri Dec 6 01:28:02 2013
@@ -72,7 +72,15 @@ public interface ILockManager
*/
public byte[] retrieveServiceData(String serviceType, String serviceName, String dataType)
throws ManifoldCFException;
-
+
+ /** Scan service data for a service type. Only active service data will be considered.
+ *@param serviceType is the type of service.
+ *@param dataType is the type of data.
+ *@param dataAcceptor is the object that will be notified of each item of data for each service name found.
+ */
+ public void scanServiceData(String serviceType, String dataType, IServiceDataAcceptor dataAcceptor)
+ throws ManifoldCFException;
+
/** Count all active services of a given type.
*@param serviceType is the service type.
*@return the count.
Added: manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IServiceDataAcceptor.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IServiceDataAcceptor.java?rev=1548357&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IServiceDataAcceptor.java (added)
+++ manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IServiceDataAcceptor.java Fri Dec 6 01:28:02 2013
@@ -0,0 +1,37 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.core.interfaces;
+
+
+/** The IServiceDataAcceptor interface describes functionality needed to
+* tally service data values across all active services of a type.
+*/
+public interface IServiceDataAcceptor
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ /** Accept service data.
+ *@param serviceName is the name of the service that owns the data.
+ *@param serviceData is the actual data that is owned.
+ *@return true to abort the scan.
+ */
+ public boolean acceptServiceData(String serviceName, byte[] serviceData)
+ throws ManifoldCFException;
+
+}
Propchange: manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IServiceDataAcceptor.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IServiceDataAcceptor.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified: manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/BaseLockManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/BaseLockManager.java?rev=1548357&r1=1548356&r2=1548357&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/BaseLockManager.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/BaseLockManager.java Fri Dec 6 01:28:02 2013
@@ -291,6 +291,41 @@ public class BaseLockManager implements
}
}
+ /** Scan service data for a service type. Only active service data will be considered.
+ *@param serviceType is the type of service.
+ *@param dataType is the type of data.
+ *@param dataAcceptor is the object that will be notified of each item of data for each service name found.
+ */
+ @Override
+ public void scanServiceData(String serviceType, String dataType, IServiceDataAcceptor dataAcceptor)
+ throws ManifoldCFException
+ {
+ String serviceTypeLockName = buildServiceTypeLockName(serviceType);
+ enterReadLock(serviceTypeLockName);
+ try
+ {
+ int i = 0;
+ while (true)
+ {
+ String resourceName = buildServiceListEntry(serviceType, i);
+ String x = readServiceName(resourceName);
+ if (x == null)
+ break;
+ if (checkGlobalFlag(makeActiveServiceFlagName(serviceType, x)))
+ {
+ byte[] serviceData = readServiceData(serviceType, x, dataType);
+ if (dataAcceptor.acceptServiceData(x, serviceData))
+ break;
+ }
+ i++;
+ }
+ }
+ finally
+ {
+ leaveReadLock(serviceTypeLockName);
+ }
+ }
+
/** Count all active services of a given type.
*@param serviceType is the service type.
*@return the count.
Modified: manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/LockManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/LockManager.java?rev=1548357&r1=1548356&r2=1548357&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/LockManager.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/LockManager.java Fri Dec 6 01:28:02 2013
@@ -96,6 +96,18 @@ public class LockManager implements ILoc
return lockManager.retrieveServiceData(serviceType, serviceName, dataType);
}
+ /** Scan service data for a service type. Only active service data will be considered.
+ *@param serviceType is the type of service.
+ *@param dataType is the type of data.
+ *@param dataAcceptor is the object that will be notified of each item of data for each service name found.
+ */
+ @Override
+ public void scanServiceData(String serviceType, String dataType, IServiceDataAcceptor dataAcceptor)
+ throws ManifoldCFException
+ {
+ lockManager.scanServiceData(serviceType, dataType, dataAcceptor);
+ }
+
/** Clean up any inactive services found.
* Calling this method will invoke cleanup of one inactive service at a time.
* If there are no inactive services around, then false will be returned.
Modified: manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java?rev=1548357&r1=1548356&r2=1548357&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperLockManager.java Fri Dec 6 01:28:02 2013
@@ -122,7 +122,7 @@ public class ZooKeeperLockManager extend
ZooKeeperConnection connection = pool.grab();
try
{
- enterServiceRegistryLock(connection, serviceType);
+ enterServiceRegistryWriteLock(connection, serviceType);
try
{
if (serviceName == null)
@@ -222,7 +222,7 @@ public class ZooKeeperLockManager extend
ZooKeeperConnection connection = pool.grab();
try
{
- enterServiceRegistryLock(connection, serviceType);
+ enterServiceRegistryWriteLock(connection, serviceType);
try
{
String dataRootPath = buildServiceTypeDataPath(serviceType, serviceName);
@@ -259,7 +259,7 @@ public class ZooKeeperLockManager extend
ZooKeeperConnection connection = pool.grab();
try
{
- enterServiceRegistryLock(connection, serviceType);
+ enterServiceRegistryReadLock(connection, serviceType);
try
{
String dataRootPath = buildServiceTypeDataPath(serviceType, serviceName);
@@ -281,6 +281,52 @@ public class ZooKeeperLockManager extend
}
}
+ /** Scan service data for a service type. Only active service data will be considered.
+ *@param serviceType is the type of service.
+ *@param dataType is the type of data.
+ *@param dataAcceptor is the object that will be notified of each item of data for each service name found.
+ */
+ @Override
+ public void scanServiceData(String serviceType, String dataType, IServiceDataAcceptor dataAcceptor)
+ throws ManifoldCFException
+ {
+ try
+ {
+ ZooKeeperConnection connection = pool.grab();
+ try
+ {
+ enterServiceRegistryReadLock(connection, serviceType);
+ try
+ {
+ String registrationNodePath = buildServiceTypeRegistrationPath(serviceType);
+ List<String> children = connection.getChildren(registrationNodePath);
+ for (String registeredServiceName : children)
+ {
+ if (connection.checkNodeExists(buildServiceTypeActivePath(serviceType, registeredServiceName)))
+ {
+ byte[] serviceData = connection.getNodeData(buildServiceTypeDataPath(serviceType, registeredServiceName), dataType);
+ if (dataAcceptor.acceptServiceData(registeredServiceName, serviceData))
+ break;
+ }
+ }
+ }
+ finally
+ {
+ leaveServiceRegistryLock(connection);
+ }
+ }
+ finally
+ {
+ pool.release(connection);
+ }
+ }
+ catch (InterruptedException e)
+ {
+ throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ }
+
+ }
+
/** Count all active services of a given type.
*@param serviceType is the service type.
*@return the count.
@@ -294,7 +340,7 @@ public class ZooKeeperLockManager extend
ZooKeeperConnection connection = pool.grab();
try
{
- enterServiceRegistryLock(connection, serviceType);
+ enterServiceRegistryReadLock(connection, serviceType);
try
{
String registrationNodePath = buildServiceTypeRegistrationPath(serviceType);
@@ -342,7 +388,7 @@ public class ZooKeeperLockManager extend
ZooKeeperConnection connection = pool.grab();
try
{
- enterServiceRegistryLock(connection, serviceType);
+ enterServiceRegistryWriteLock(connection, serviceType);
try
{
// We find ONE service that is registered but inactive, and clean up after that one.
@@ -402,7 +448,7 @@ public class ZooKeeperLockManager extend
ZooKeeperConnection connection = pool.grab();
try
{
- enterServiceRegistryLock(connection, serviceType);
+ enterServiceRegistryWriteLock(connection, serviceType);
try
{
connection.deleteNodeChildren(buildServiceTypeDataPath(serviceType,serviceName));
@@ -440,7 +486,7 @@ public class ZooKeeperLockManager extend
ZooKeeperConnection connection = pool.grab();
try
{
- enterServiceRegistryLock(connection, serviceType);
+ enterServiceRegistryReadLock(connection, serviceType);
try
{
return connection.checkNodeExists(buildServiceTypeActivePath(serviceType, serviceName));
@@ -461,13 +507,27 @@ public class ZooKeeperLockManager extend
}
}
- /** Enter service registry lock */
- protected void enterServiceRegistryLock(ZooKeeperConnection connection, String serviceType)
+ /** Enter service registry read lock */
+ protected void enterServiceRegistryReadLock(ZooKeeperConnection connection, String serviceType)
+ throws ManifoldCFException, InterruptedException
+ {
+ String serviceTypeLock = buildServiceTypeLockPath(serviceType);
+ while (true)
+ {
+ if (connection.obtainReadLockNoWait(serviceTypeLock))
+ return;
+ ManifoldCF.sleep(100L);
+ }
+ }
+
+ /** Enter service registry write lock */
+ protected void enterServiceRegistryWriteLock(ZooKeeperConnection connection, String serviceType)
throws ManifoldCFException, InterruptedException
{
+ String serviceTypeLock = buildServiceTypeLockPath(serviceType);
while (true)
{
- if (connection.obtainWriteLockNoWait(buildServiceTypeLockPath(serviceType)))
+ if (connection.obtainWriteLockNoWait(serviceTypeLock))
return;
ManifoldCF.sleep(100L);
}