You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/12/02 14:45:49 UTC
svn commit: r1547019 - in /manifoldcf/branches/CONNECTORS-781/framework:
agents/src/main/java/org/apache/manifoldcf/agents/
agents/src/main/java/org/apache/manifoldcf/agents/system/
combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ ...
Author: kwright
Date: Mon Dec 2 13:45:49 2013
New Revision: 1547019
URL: http://svn.apache.org/r1547019
Log:
Switch over to using the instantiable class for agent daemon management.
Modified:
manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java
manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java
manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java
manifoldcf/branches/CONNECTORS-781/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java
manifoldcf/branches/CONNECTORS-781/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java
manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java
Modified: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java?rev=1547019&r1=1547018&r2=1547019&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java Mon Dec 2 13:45:49 2013
@@ -59,8 +59,9 @@ public class AgentRun extends BaseAgents
Logging.root.info("Running...");
// Register hook first so stopAgents() not required
- ManifoldCF.registerAgentsShutdownHook(tc, processID);
- ManifoldCF.runAgents(tc, processID);
+ AgentsDaemon ad = new AgentsDaemon(processID);
+ ad.registerAgentsShutdownHook(tc);
+ ad.runAgents(tc);
Logging.root.info("Shutting down...");
}
catch (ManifoldCFException e)
Modified: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java?rev=1547019&r1=1547018&r2=1547019&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java Mon Dec 2 13:45:49 2013
@@ -37,7 +37,7 @@ public class AgentStop extends BaseAgent
// As part of the work for CONNECTORS-781, this method is now synchronous.
// We assert the shutdown signal, and then wait until all active services have shut down.
ILockManager lockManager = LockManagerFactory.make(tc);
- ManifoldCF.assertAgentsShutdownSignal(tc);
+ AgentsDaemon.assertAgentsShutdownSignal(tc);
try
{
Logging.root.info("Shutdown signal sent");
@@ -61,7 +61,7 @@ public class AgentStop extends BaseAgent
finally
{
// Clear shutdown signal
- ManifoldCF.clearAgentsShutdownSignal(tc);
+ AgentsDaemon.clearAgentsShutdownSignal(tc);
}
}
Modified: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java?rev=1547019&r1=1547018&r2=1547019&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java Mon Dec 2 13:45:49 2013
@@ -32,11 +32,6 @@ public class ManifoldCF extends org.apac
// Agents initialized flag
protected static boolean agentsInitialized = false;
- /** This is the place we keep track of the agents we've started. */
- protected static Map<String,IAgent> runningHash = new HashMap<String,IAgent>();
- /** This flag prevents startAgents() from starting anything once stopAgents() has been called. */
- protected static boolean stopAgentsRun = false;
-
/** Initialize environment.
*/
public static void initializeEnvironment(IThreadContext threadContext)
@@ -88,10 +83,6 @@ public class ManifoldCF extends org.apac
synchronized (initializeFlagLock)
{
org.apache.manifoldcf.core.system.ManifoldCF.resetEnvironment(threadContext);
- synchronized (runningHash)
- {
- stopAgentsRun = false;
- }
}
}
@@ -128,317 +119,6 @@ public class ManifoldCF extends org.apac
mgr.deinstall();
}
- // There are a number of different ways of running the agents framework.
- // (1) Repeatedly call checkAgents(), and when all done make sure to call stopAgents().
- // (2) Call registerAgentsShutdownHook(), then repeatedly run checkAgents(), Agent shutdown happens on JVM exit.
- // (3) Call runAgents(), which will wait for someone else to call assertAgentsShutdownSignal(). Before exit, stopAgents() must be called.
- // (4) Call registerAgentsShutdownHook(), then call runAgents(), which will wait for someone else to call assertAgentsShutdownSignal(). Shutdown happens on JVM exit.
-
- /** Assert shutdown signal.
- */
- public static void assertAgentsShutdownSignal(IThreadContext threadContext)
- throws ManifoldCFException
- {
- ILockManager lockManager = LockManagerFactory.make(threadContext);
- lockManager.setGlobalFlag(agentShutdownSignal);
- }
-
- /** Clear shutdown signal.
- */
- public static void clearAgentsShutdownSignal(IThreadContext threadContext)
- throws ManifoldCFException
- {
- ILockManager lockManager = LockManagerFactory.make(threadContext);
- lockManager.clearGlobalFlag(agentShutdownSignal);
- }
-
-
- /** Register agents shutdown hook.
- * Call this ONCE before calling startAgents or checkAgents the first time, if you want automatic cleanup of agents on JVM stop.
- */
- public static void registerAgentsShutdownHook(IThreadContext threadContext, String processID)
- throws ManifoldCFException
- {
- // Create the shutdown hook for agents. All activity will be keyed off of runningHash, so it is safe to do this under all conditions.
- org.apache.manifoldcf.core.system.ManifoldCF.addShutdownHook(new AgentsShutdownHook(processID));
- }
-
- /** Agent service name prefix (followed by agent class name) */
- public static final String agentServicePrefix = "AGENT_";
-
- protected static AgentsThread agentsThread = null;
-
- /** Run agents process.
- * This method will not return until a shutdown signal is sent.
- */
- public static void runAgents(IThreadContext threadContext, String processID)
- throws ManifoldCFException
- {
- ILockManager lockManager = LockManagerFactory.make(threadContext);
-
- // Don't come up at all if shutdown signal in force
- if (lockManager.checkGlobalFlag(agentShutdownSignal))
- return;
-
- // Create and start agents thread.
- startAgents(threadContext, processID);
-
- while (true)
- {
- // Any shutdown signal yet?
- if (lockManager.checkGlobalFlag(agentShutdownSignal))
- break;
-
- try
- {
- ManifoldCF.sleep(5000L);
- }
- catch (InterruptedException e)
- {
- break;
- }
- }
-
- }
-
- /** Start agents thread.
- */
- public static void startAgents(IThreadContext threadContext, String processID)
- throws ManifoldCFException
- {
- // Create and start agents thread.
- agentsThread = new AgentsThread(processID);
- agentsThread.start();
- }
-
- /** Stop all started agents.
- */
- public static void stopAgents(IThreadContext threadContext, String processID)
- throws ManifoldCFException
- {
- // Shut down agents background thread.
- while (agentsThread != null)
- {
- agentsThread.interrupt();
- if (!agentsThread.isAlive())
- agentsThread = null;
- }
-
- // Shut down running agents services directly.
- ILockManager lockManager = LockManagerFactory.make(threadContext);
- synchronized (runningHash)
- {
- // This is supposedly safe; iterator remove is used
- Iterator<String> iter = runningHash.keySet().iterator();
- while (iter.hasNext())
- {
- String className = iter.next();
- IAgent agent = runningHash.get(className);
- // Stop it
- agent.stopAgent(threadContext);
- lockManager.endServiceActivity(getAgentsClassServiceType(className), processID);
- iter.remove();
- agent.cleanUp(threadContext);
- }
- }
- // Done.
- }
-
- protected static String getAgentsClassServiceType(String agentClassName)
- {
- return agentServicePrefix + agentClassName;
- }
-
- /** Agents thread. This runs in background until interrupted, at which point
- * it shuts down. Its responsibilities include cleaning up after dead processes,
- * as well as starting newly-registered agent processes, and terminating ones that disappear.
- */
- protected static class AgentsThread extends Thread
- {
- protected final String processID;
-
- public AgentsThread(String processID)
- {
- super();
- this.processID = processID;
- setName("Agents thread");
- setDaemon(true);
- }
-
- public void run()
- {
- try
- {
- IThreadContext threadContext = ThreadContextFactory.make();
- while (true)
- {
- try
- {
- if (Thread.currentThread().isInterrupted())
- throw new ManifoldCFException("Interrupted",ManifoldCFException.INTERRUPTED);
-
- checkAgents(threadContext, processID);
- ManifoldCF.sleep(5000L);
- }
- catch (InterruptedException e)
- {
- break;
- }
- catch (ManifoldCFException e)
- {
- if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
- break;
- Logging.agents.error("Exception tossed: "+e.getMessage(),e);
- }
- catch (OutOfMemoryError e)
- {
- System.err.println("Agents process ran out of memory - shutting down");
- e.printStackTrace(System.err);
- System.exit(-200);
- }
- catch (Throwable e)
- {
- Logging.agents.fatal("Error tossed: "+e.getMessage(),e);
- }
- }
- }
- catch (Throwable e)
- {
- // Severe error on initialization
- System.err.println("Agents process could not start - shutting down");
- Logging.agents.fatal("AgentThread initialization error tossed: "+e.getMessage(),e);
- System.exit(-300);
- }
- }
- }
-
- /** Start all not-running agents.
- *@param threadContext is the thread context.
- */
- protected static void checkAgents(IThreadContext threadContext, String processID)
- throws ManifoldCFException
- {
- ILockManager lockManager = LockManagerFactory.make(threadContext);
- // Get agent manager
- IAgentManager manager = AgentManagerFactory.make(threadContext);
- synchronized (runningHash)
- {
- String[] classes = manager.getAllAgents();
- Set<String> currentAgentClasses = new HashSet<String>();
-
- int i = 0;
- while (i < classes.length)
- {
- String className = classes[i++];
- if (runningHash.get(className) == null)
- {
- // Start this agent
- IAgent agent = AgentFactory.make(className);
- agent.initialize(threadContext);
- try
- {
- // Throw a lock, so that cleanup processes and startup processes don't collide.
- String serviceType = getAgentsClassServiceType(className);
- lockManager.registerServiceBeginServiceActivity(serviceType, processID, new CleanupAgent(threadContext, agent, processID));
- // There is a potential race condition where the agent has been started but hasn't yet appeared in runningHash.
- // But having runningHash be the synchronizer for this activity will prevent any problems.
- agent.startAgent(threadContext, processID);
- // Successful!
- runningHash.put(className,agent);
- }
- catch (ManifoldCFException e)
- {
- if (e.getErrorCode() != ManifoldCFException.INTERRUPTED)
- agent.cleanUp(threadContext);
- throw e;
- }
- }
- currentAgentClasses.add(className);
- }
-
- // Go through running hash and look for agents processes that have left
- Iterator<String> runningAgentsIterator = runningHash.keySet().iterator();
- while (runningAgentsIterator.hasNext())
- {
- String runningAgentClass = runningAgentsIterator.next();
- if (!currentAgentClasses.contains(runningAgentClass))
- {
- // Shut down this one agent.
- IAgent agent = runningHash.get(runningAgentClass);
- // Stop it
- agent.stopAgent(threadContext);
- lockManager.endServiceActivity(getAgentsClassServiceType(runningAgentClass), processID);
- runningAgentsIterator.remove();
- agent.cleanUp(threadContext);
- }
- }
- }
-
- synchronized (runningHash)
- {
- // For every class we're supposed to be running, find registered but no-longer-active instances and clean
- // up after them.
- for (String agentsClass : runningHash.keySet())
- {
- IAgent agent = runningHash.get(agentsClass);
- IServiceCleanup cleanup = new CleanupAgent(threadContext, agent, processID);
- String agentsClassServiceType = getAgentsClassServiceType(agentsClass);
- while (!lockManager.cleanupInactiveService(agentsClassServiceType, cleanup))
- {
- // Loop until no more inactive services
- }
- }
- }
-
- }
-
-
- protected static class CleanupAgent implements IServiceCleanup
- {
- protected final IAgent agent;
- protected final IThreadContext threadContext;
- protected final String processID;
-
- public CleanupAgent(IThreadContext threadContext, IAgent agent, String processID)
- {
- this.agent = agent;
- this.threadContext = threadContext;
- this.processID = processID;
- }
-
- /** Clean up after the specified service. This method will block any startup of the specified
- * service for as long as it runs.
- *@param serviceName is the name of the service.
- */
- @Override
- public void cleanUpService(String serviceName)
- throws ManifoldCFException
- {
- agent.cleanUpAgentData(threadContext, processID, serviceName);
- }
-
- /** Clean up after ALL services of the type on the cluster.
- */
- @Override
- public void cleanUpAllServices()
- throws ManifoldCFException
- {
- agent.cleanUpAllAgentData(threadContext, processID);
- }
-
- /** Perform cluster initialization - that is, whatever is needed presuming that the
- * cluster has been down for an indeterminate period of time, but is otherwise in a clean
- * state.
- */
- @Override
- public void clusterInit()
- throws ManifoldCFException
- {
- agent.clusterInit(threadContext);
- }
-
- }
-
/** Signal output connection needs redoing.
* This is called when something external changed on an output connection, and
* therefore all associated documents must be reindexed.
@@ -456,30 +136,6 @@ public class ManifoldCF extends org.apac
AgentManagerFactory.noteOutputConnectionChange(threadContext,connectionName);
}
- /** Agents shutdown hook class */
- protected static class AgentsShutdownHook implements IShutdownHook
- {
- protected final String processID;
-
- public AgentsShutdownHook(String processID)
- {
- this.processID = processID;
- }
-
- public void doCleanup()
- throws ManifoldCFException
- {
- // Shutting down in this way must prevent startup from taking place.
- synchronized (runningHash)
- {
- stopAgentsRun = true;
- }
- IThreadContext tc = ThreadContextFactory.make();
- stopAgents(tc,processID);
- }
-
- }
-
// Helper methods for API support. These are made public so connectors can use them to implement the executeCommand method.
// These are the universal node types.
Modified: manifoldcf/branches/CONNECTORS-781/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java?rev=1547019&r1=1547018&r2=1547019&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java Mon Dec 2 13:45:49 2013
@@ -20,6 +20,7 @@ package org.apache.manifoldcf.combinedse
import org.apache.manifoldcf.core.interfaces.*;
import org.apache.manifoldcf.crawler.system.ManifoldCF;
+import org.apache.manifoldcf.agents.system.AgentsDaemon;
import javax.servlet.*;
/** This class furnishes a servlet shutdown hook for ManifoldCF. It should be referenced in the
@@ -59,10 +60,10 @@ public class ServletListener implements
{
if (agentsThread != null)
{
- ManifoldCF.assertAgentsShutdownSignal(tc);
+ AgentsDaemon.assertAgentsShutdownSignal(tc);
agentsThread.finishUp();
agentsThread = null;
- ManifoldCF.clearAgentsShutdownSignal(tc);
+ AgentsDaemon.clearAgentsShutdownSignal(tc);
}
}
catch (InterruptedException e)
@@ -94,14 +95,15 @@ public class ServletListener implements
IThreadContext tc = ThreadContextFactory.make();
try
{
- ManifoldCF.clearAgentsShutdownSignal(tc);
+ AgentsDaemon.clearAgentsShutdownSignal(tc);
+ AgentsDaemon ad = new AgentsDaemon(processID);
try
{
- ManifoldCF.runAgents(tc, processID);
+ ad.runAgents(tc);
}
finally
{
- ManifoldCF.stopAgents(tc, processID);
+ ad.stopAgents(tc);
}
}
catch (Throwable e)
Modified: manifoldcf/branches/CONNECTORS-781/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java?rev=1547019&r1=1547018&r2=1547019&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java Mon Dec 2 13:45:49 2013
@@ -21,6 +21,7 @@ import java.io.*;
import org.apache.manifoldcf.core.interfaces.*;
import org.apache.manifoldcf.crawler.system.*;
import org.apache.manifoldcf.crawler.*;
+import org.apache.manifoldcf.agents.system.AgentsDaemon;
import java.io.IOException;
import java.io.InputStream;
@@ -138,8 +139,9 @@ public class ManifoldCFJettyRunner
{
String processID = ManifoldCF.getProcessID();
// Do this so we don't have to call stopAgents() ourselves.
- ManifoldCF.registerAgentsShutdownHook(tc, processID);
- ManifoldCF.runAgents(tc, processID);
+ AgentsDaemon ad = new AgentsDaemon(processID);
+ ad.registerAgentsShutdownHook(tc);
+ ad.runAgents(tc);
}
/**
@@ -198,7 +200,7 @@ public class ManifoldCFJettyRunner
if (useParentClassLoader)
{
// Clear the agents shutdown signal.
- ManifoldCF.clearAgentsShutdownSignal(tc);
+ AgentsDaemon.clearAgentsShutdownSignal(tc);
// Do the basic initialization of the database and its schema
ManifoldCF.createSystemDatabase(tc);
Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java?rev=1547019&r1=1547018&r2=1547019&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java (original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java Mon Dec 2 13:45:49 2013
@@ -22,6 +22,7 @@ import org.apache.manifoldcf.core.interf
import org.apache.manifoldcf.agents.interfaces.*;
import org.apache.manifoldcf.crawler.interfaces.*;
import org.apache.manifoldcf.crawler.system.ManifoldCF;
+import org.apache.manifoldcf.agents.system.AgentsDaemon;
import java.io.*;
import java.util.*;
@@ -575,7 +576,7 @@ public class ManifoldCFInstance
// If all worked, then we can start the daemon.
// Clear the agents shutdown signal.
IThreadContext tc = ThreadContextFactory.make();
- ManifoldCF.clearAgentsShutdownSignal(tc);
+ AgentsDaemon.clearAgentsShutdownSignal(tc);
daemonThread = new DaemonThread();
daemonThread.start();
@@ -670,7 +671,7 @@ public class ManifoldCFInstance
if (!singleWar)
{
// Shut down daemon
- ManifoldCF.assertAgentsShutdownSignal(tc);
+ AgentsDaemon.assertAgentsShutdownSignal(tc);
// Wait for daemon thread to exit.
while (true)
@@ -721,9 +722,10 @@ public class ManifoldCFInstance
IThreadContext tc = ThreadContextFactory.make();
// Now, start the server, and then wait for the shutdown signal. On shutdown, we have to actually do the cleanup,
// because the JVM isn't going away.
+ AgentsDaemon ad = new AgentsDaemon(processID);
try
{
- ManifoldCF.runAgents(tc, processID);
+ ad.runAgents(tc);
}
catch (ManifoldCFException e)
{
@@ -733,7 +735,7 @@ public class ManifoldCFInstance
{
try
{
- ManifoldCF.stopAgents(tc, processID);
+ ad.stopAgents(tc);
}
catch (ManifoldCFException e)
{