You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/12/04 15:08:50 UTC
svn commit: r1547801 - in /manifoldcf/branches/CONNECTORS-781/framework:
agents/src/main/java/org/apache/manifoldcf/agents/system/
pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/
Author: kwright
Date: Wed Dec 4 14:08:50 2013
New Revision: 1547801
URL: http://svn.apache.org/r1547801
Log:
Add agents-level idle cleanup thread
Added:
manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java (with props)
Modified:
manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/AgentsDaemon.java
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java
Modified: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/AgentsDaemon.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/AgentsDaemon.java?rev=1547801&r1=1547800&r2=1547801&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/AgentsDaemon.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/AgentsDaemon.java Wed Dec 4 14:08:50 2013
@@ -36,6 +36,9 @@ public class AgentsDaemon
* also takes on process cleanup where necessary. */
protected AgentsThread agentsThread = null;
+ /** The idle cleanup thread. */
+ protected IdleCleanupThread idleCleanupThread = null;
+
/** Process ID for this agents daemon. */
protected final String processID;
@@ -124,8 +127,11 @@ public class AgentsDaemon
public void startAgents(IThreadContext threadContext)
throws ManifoldCFException
{
- // Create and start agents thread.
+ // Create idle cleanup thread.
+ idleCleanupThread = new IdleCleanupThread(processID);
agentsThread = new AgentsThread();
+ // Create and start agents thread.
+ idleCleanupThread.start();
agentsThread.start();
}
@@ -135,11 +141,17 @@ public class AgentsDaemon
throws ManifoldCFException
{
// Shut down agents background thread.
- while (agentsThread != null)
+ while (agentsThread != null || idleCleanupThread != null)
{
- agentsThread.interrupt();
- if (!agentsThread.isAlive())
+ if (agentsThread != null)
+ agentsThread.interrupt();
+ if (idleCleanupThread != null)
+ idleCleanupThread.interrupt();
+
+ if (agentsThread != null && !agentsThread.isAlive())
agentsThread = null;
+ if (idleCleanupThread != null && !idleCleanupThread.isAlive())
+ idleCleanupThread = null;
}
// Shut down running agents services directly.
Added: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java?rev=1547801&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java (added)
+++ manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java Wed Dec 4 14:08:50 2013
@@ -0,0 +1,130 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.system;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import java.util.*;
+
+/** This thread periodically calls the cleanup method in all connected output connectors. The ostensible purpose
+* is to allow the connectors to shutdown idle connections etc.
+*/
+public class IdleCleanupThread extends Thread
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ // Local data
+ /** Process ID */
+ protected final String processID;
+
+ /** Constructor.
+ */
+ public IdleCleanupThread(String processID)
+ throws ManifoldCFException
+ {
+ super();
+ this.processID = processID;
+ setName("Idle cleanup thread");
+ setDaemon(true);
+ }
+
+ public void run()
+ {
+ Logging.agents.debug("Start up idle cleanup thread");
+ try
+ {
+ // Create a thread context object.
+ IThreadContext threadContext = ThreadContextFactory.make();
+ // Get the cache handle.
+ ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
+ // Get the output connector pool handle
+ IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(threadContext);
+
+ // Loop
+ while (true)
+ {
+ // Do another try/catch around everything in the loop
+ try
+ {
+ // Do the cleanup
+ outputConnectorPool.pollAllConnectors();
+ cacheManager.expireObjects(System.currentTimeMillis());
+
+ // Sleep for the retry interval.
+ ManifoldCF.sleep(15000L);
+ }
+ catch (ManifoldCFException e)
+ {
+ if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+ break;
+
+ if (e.getErrorCode() == ManifoldCFException.DATABASE_CONNECTION_ERROR)
+ {
+ Logging.agents.error("Idle cleanup thread aborting and restarting due to database connection reset: "+e.getMessage(),e);
+ try
+ {
+ // Give the database a chance to catch up/wake up
+ ManifoldCF.sleep(10000L);
+ }
+ catch (InterruptedException se)
+ {
+ break;
+ }
+ continue;
+ }
+
+ // Log it, but keep the thread alive
+ Logging.agents.error("Exception tossed: "+e.getMessage(),e);
+
+ if (e.getErrorCode() == ManifoldCFException.SETUP_ERROR)
+ {
+ // Shut the whole system down!
+ System.exit(1);
+ }
+
+ }
+ catch (InterruptedException e)
+ {
+ // We're supposed to quit
+ break;
+ }
+ catch (OutOfMemoryError e)
+ {
+ System.err.println("agents process ran out of memory - shutting down");
+ e.printStackTrace(System.err);
+ System.exit(-200);
+ }
+ catch (Throwable e)
+ {
+ // A more severe error - but stay alive
+ Logging.agents.fatal("Error tossed: "+e.getMessage(),e);
+ }
+ }
+ }
+ catch (Throwable e)
+ {
+ // Severe error on initialization
+ System.err.println("agents process could not start - shutting down");
+ Logging.agents.fatal("IdleCleanupThread initialization error tossed: "+e.getMessage(),e);
+ System.exit(-300);
+ }
+
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java?rev=1547801&r1=1547800&r2=1547801&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java Wed Dec 4 14:08:50 2013
@@ -56,8 +56,6 @@ public class IdleCleanupThread extends T
IThreadContext threadContext = ThreadContextFactory.make();
// Get the cache handle.
ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
- // Get the output connector pool handle
- IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(threadContext);
// Loop
while (true)
@@ -67,7 +65,6 @@ public class IdleCleanupThread extends T
{
// Do the cleanup
RepositoryConnectorFactory.pollAllConnectors(threadContext);
- outputConnectorPool.pollAllConnectors();
cacheManager.expireObjects(System.currentTimeMillis());
// Sleep for the retry interval.