You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/11/16 18:45:02 UTC
svn commit: r1640018 - in /manifoldcf/trunk/framework:
agents/src/main/java/org/apache/manifoldcf/agents/system/
api-service/src/main/java/org/apache/manifoldcf/apiservice/
combined-service/src/main/java/org/apache/manifoldcf/combinedservice/
core/src/...
Author: kwright
Date: Sun Nov 16 17:45:01 2014
New Revision: 1640018
URL: http://svn.apache.org/r1640018
Log:
Create general-purpose service for registering polling events.
Added:
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IPollingHook.java (with props)
Modified:
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
manifoldcf/trunk/framework/api-service/src/main/java/org/apache/manifoldcf/apiservice/IdleCleanupThread.java
manifoldcf/trunk/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/IdleCleanupThread.java
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
manifoldcf/trunk/framework/crawler-ui/src/main/java/org/apache/manifoldcf/crawlerui/IdleCleanupThread.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/system/IdleCleanupThread.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java
Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java?rev=1640018&r1=1640017&r2=1640018&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java (original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java Sun Nov 16 17:45:01 2014
@@ -51,14 +51,10 @@ public class IdleCleanupThread extends T
{
// Create a thread context object.
IThreadContext threadContext = ThreadContextFactory.make();
- // Get the cache handle.
- ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
// Get the output connector pool handle
IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(threadContext);
// Get the transformation connector pool handle
ITransformationConnectorPool transformationConnectorPool = TransformationConnectorPoolFactory.make(threadContext);
- // Throttler subsystem
- IThrottleGroups throttleGroups = ThrottleGroupsFactory.make(threadContext);
/* For HSQLDB debugging...
IDBInterface database = DBInterfaceFactory.make(threadContext,
@@ -93,10 +89,8 @@ public class IdleCleanupThread extends T
// Do the cleanup
outputConnectorPool.pollAllConnectors();
transformationConnectorPool.pollAllConnectors();
- // Poll connection bins
- throttleGroups.poll();
- // Expire objects
- cacheManager.expireObjects(System.currentTimeMillis());
+ // Poll all basic services
+ ManifoldCF.pollAll(threadContext);
// Sleep for the retry interval.
ManifoldCF.sleep(5000L);
Modified: manifoldcf/trunk/framework/api-service/src/main/java/org/apache/manifoldcf/apiservice/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/api-service/src/main/java/org/apache/manifoldcf/apiservice/IdleCleanupThread.java?rev=1640018&r1=1640017&r2=1640018&view=diff
==============================================================================
--- manifoldcf/trunk/framework/api-service/src/main/java/org/apache/manifoldcf/apiservice/IdleCleanupThread.java (original)
+++ manifoldcf/trunk/framework/api-service/src/main/java/org/apache/manifoldcf/apiservice/IdleCleanupThread.java Sun Nov 16 17:45:01 2014
@@ -51,16 +51,12 @@ public class IdleCleanupThread extends T
{
// Create a thread context object.
IThreadContext threadContext = ThreadContextFactory.make();
- // Get the cache handle.
- ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(threadContext);
ITransformationConnectorPool transformationConnectorPool = TransformationConnectorPoolFactory.make(threadContext);
IAuthorityConnectorPool authorityConnectorPool = AuthorityConnectorPoolFactory.make(threadContext);
IMappingConnectorPool mappingConnectorPool = MappingConnectorPoolFactory.make(threadContext);
-
- IThrottleGroups throttleGroups = ThrottleGroupsFactory.make(threadContext);
// Loop
while (true)
@@ -74,10 +70,8 @@ public class IdleCleanupThread extends T
transformationConnectorPool.pollAllConnectors();
authorityConnectorPool.pollAllConnectors();
mappingConnectorPool.pollAllConnectors();
-
- throttleGroups.poll();
-
- cacheManager.expireObjects(System.currentTimeMillis());
+ // Poll all basic services
+ ManifoldCF.pollAll(threadContext);
// Sleep for the retry interval.
ManifoldCF.sleep(5000L);
Modified: manifoldcf/trunk/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/IdleCleanupThread.java?rev=1640018&r1=1640017&r2=1640018&view=diff
==============================================================================
--- manifoldcf/trunk/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/IdleCleanupThread.java (original)
+++ manifoldcf/trunk/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/IdleCleanupThread.java Sun Nov 16 17:45:01 2014
@@ -51,16 +51,12 @@ public class IdleCleanupThread extends T
{
// Create a thread context object.
IThreadContext threadContext = ThreadContextFactory.make();
- // Get the cache handle.
- ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(threadContext);
ITransformationConnectorPool transformationConnectorPool = TransformationConnectorPoolFactory.make(threadContext);
IAuthorityConnectorPool authorityConnectorPool = AuthorityConnectorPoolFactory.make(threadContext);
IMappingConnectorPool mappingConnectorPool = MappingConnectorPoolFactory.make(threadContext);
-
- IThrottleGroups throttleGroups = ThrottleGroupsFactory.make(threadContext);
// Loop
while (true)
@@ -74,11 +70,9 @@ public class IdleCleanupThread extends T
transformationConnectorPool.pollAllConnectors();
authorityConnectorPool.pollAllConnectors();
mappingConnectorPool.pollAllConnectors();
-
- throttleGroups.poll();
-
- cacheManager.expireObjects(System.currentTimeMillis());
-
+ // Poll all basic services
+ ManifoldCF.pollAll(threadContext);
+
// Sleep for the retry interval.
ManifoldCF.sleep(5000L);
}
Added: manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IPollingHook.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IPollingHook.java?rev=1640018&view=auto
==============================================================================
--- manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IPollingHook.java (added)
+++ manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IPollingHook.java Sun Nov 16 17:45:01 2014
@@ -0,0 +1,29 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.core.interfaces;
+
+/** This interface describes functionality needed for a periodic polling operation */
+public interface IPollingHook
+{
+ /** Do the requisite cleanup.
+ */
+ public void doPoll(IThreadContext threadContext)
+ throws ManifoldCFException;
+}
+
Propchange: manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IPollingHook.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IPollingHook.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified: manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java?rev=1640018&r1=1640017&r2=1640018&view=diff
==============================================================================
--- manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java (original)
+++ manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java Sun Nov 16 17:45:01 2014
@@ -72,7 +72,10 @@ public class ManifoldCF
protected static DatabaseShutdown dbShutdown = null;
/** Array of cleanup hooks (for managing shutdown) */
- protected static ArrayList cleanupHooks = new ArrayList();
+ protected final static List<IShutdownHook> cleanupHooks = new ArrayList<IShutdownHook>();
+
+ /** Array of polling hooks (for managing polling) */
+ protected final static List<IPollingHook> pollingHooks = new ArrayList<IPollingHook>();
/** Shutdown thread */
protected static Thread shutdownThread;
@@ -275,8 +278,13 @@ public class ManifoldCF
masterDatabaseUsername = LockManagerFactory.getStringProperty(threadContext,masterDatabaseUsernameProperty,"manifoldcf");
masterDatabasePassword = LockManagerFactory.getPossiblyObfuscatedStringProperty(threadContext,masterDatabasePasswordProperty,"local_pg_passwd");
- // Register the throttler for cleanup on shutdown
+ // Register the throttler
addShutdownHook(new ThrottlerShutdown());
+ addPollingHook(new ThrottlerPoll());
+
+ // Put the cache manager in the polling loop
+ addPollingHook(new CachePoll());
+
// Register the file tracker for cleanup on shutdown
tracker = new FileTrack();
addShutdownHook(tracker);
@@ -1423,6 +1431,29 @@ public class ManifoldCF
cleanupHooks.add(hook);
}
}
+
+ /** Add a polling hook to the list. These hooks will be evaluated in the
+ * order they were added.
+ *@param hook is the polling hook that needs to be added to the sequence.
+ */
+ public static void addPollingHook(IPollingHook hook)
+ {
+ synchronized (pollingHooks)
+ {
+ pollingHooks.add(hook);
+ }
+ }
+
+ /** Poll all the registered polling services.
+ */
+ public static void pollAll(IThreadContext threadContext)
+ throws ManifoldCFException
+ {
+ for (IPollingHook hook : pollingHooks)
+ {
+ hook.doPoll(threadContext);
+ }
+ }
/** Create a new resource loader based on the default one. This is used by
* connectors wishing to make their own resource loaders for isolation purposes.
@@ -1465,7 +1496,7 @@ public class ManifoldCF
while (i > 0)
{
i--;
- IShutdownHook hook = (IShutdownHook)cleanupHooks.get(i);
+ IShutdownHook hook = cleanupHooks.get(i);
try
{
hook.doCleanup(threadContext);
@@ -1548,6 +1579,22 @@ public class ManifoldCF
}
+ /** Class that polls throttler */
+ protected static class ThrottlerPoll implements IPollingHook
+ {
+ public ThrottlerPoll()
+ {
+ }
+
+ @Override
+ public void doPoll(IThreadContext threadContext)
+ throws ManifoldCFException
+ {
+ IThrottleGroups connectionThrottler = ThrottleGroupsFactory.make(threadContext);
+ connectionThrottler.poll();
+ }
+ }
+
/** Class that cleans up throttler on exit */
protected static class ThrottlerShutdown implements IShutdownHook
{
@@ -1580,6 +1627,23 @@ public class ManifoldCF
}
+ /** Class that cleans up expired cache objects on polling.
+ */
+ protected static class CachePoll implements IPollingHook
+ {
+ public CachePoll()
+ {
+ }
+
+ @Override
+ public void doPoll(IThreadContext threadContext)
+ throws ManifoldCFException
+ {
+ ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
+ cacheManager.expireObjects(System.currentTimeMillis());
+ }
+ }
+
/** Class that cleans up database handles on exit */
protected static class DatabaseShutdown implements IShutdownHook
{
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/java/org/apache/manifoldcf/crawlerui/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/java/org/apache/manifoldcf/crawlerui/IdleCleanupThread.java?rev=1640018&r1=1640017&r2=1640018&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/java/org/apache/manifoldcf/crawlerui/IdleCleanupThread.java (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/java/org/apache/manifoldcf/crawlerui/IdleCleanupThread.java Sun Nov 16 17:45:01 2014
@@ -51,16 +51,12 @@ public class IdleCleanupThread extends T
{
// Create a thread context object.
IThreadContext threadContext = ThreadContextFactory.make();
- // Get the cache handle.
- ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(threadContext);
ITransformationConnectorPool transformationConnectorPool = TransformationConnectorPoolFactory.make(threadContext);
IAuthorityConnectorPool authorityConnectorPool = AuthorityConnectorPoolFactory.make(threadContext);
IMappingConnectorPool mappingConnectorPool = MappingConnectorPoolFactory.make(threadContext);
-
- IThrottleGroups throttleGroups = ThrottleGroupsFactory.make(threadContext);
// Loop
while (true)
@@ -75,9 +71,8 @@ public class IdleCleanupThread extends T
authorityConnectorPool.pollAllConnectors();
mappingConnectorPool.pollAllConnectors();
- throttleGroups.poll();
-
- cacheManager.expireObjects(System.currentTimeMillis());
+ // Poll all basic services
+ ManifoldCF.pollAll(threadContext);
// Sleep for the retry interval.
ManifoldCF.sleep(5000L);
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/system/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/system/IdleCleanupThread.java?rev=1640018&r1=1640017&r2=1640018&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/system/IdleCleanupThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/system/IdleCleanupThread.java Sun Nov 16 17:45:01 2014
@@ -51,7 +51,6 @@ public class IdleCleanupThread extends T
{
// Create a thread context object.
IThreadContext threadContext = ThreadContextFactory.make();
- ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
IAuthorityConnectorPool authorityConnectorPool = AuthorityConnectorPoolFactory.make(threadContext);
IMappingConnectorPool mappingConnectorPool = MappingConnectorPoolFactory.make(threadContext);
@@ -64,7 +63,8 @@ public class IdleCleanupThread extends T
// Do the cleanup
authorityConnectorPool.pollAllConnectors();
mappingConnectorPool.pollAllConnectors();
- cacheManager.expireObjects(System.currentTimeMillis());
+ // Poll all basic services
+ ManifoldCF.pollAll(threadContext);
// Sleep for the retry interval.
ManifoldCF.sleep(5000L);
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java?rev=1640018&r1=1640017&r2=1640018&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java Sun Nov 16 17:45:01 2014
@@ -54,8 +54,6 @@ public class IdleCleanupThread extends T
{
// Create a thread context object.
IThreadContext threadContext = ThreadContextFactory.make();
- // Get the cache handle.
- ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
@@ -67,7 +65,7 @@ public class IdleCleanupThread extends T
{
// Do the cleanup
repositoryConnectorPool.pollAllConnectors();
- cacheManager.expireObjects(System.currentTimeMillis());
+ ManifoldCF.pollAll(threadContext);
// Sleep for the retry interval.
ManifoldCF.sleep(5000L);