You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/12/16 03:00:08 UTC
svn commit: r1645816 - in
/manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler:
interfaces/IJobManager.java
interfaces/NotificationConnectionManagerFactory.java jobs/JobManager.java
system/ManifoldCF.java
Author: kwright
Date: Tue Dec 16 02:00:08 2014
New Revision: 1645816
URL: http://svn.apache.org/r1645816
Log:
Hook up notification connection and connector management
Added:
manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/NotificationConnectionManagerFactory.java (with props)
Modified:
manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java
manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
Modified: manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java?rev=1645816&r1=1645815&r2=1645816&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java (original)
+++ manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java Tue Dec 16 02:00:08 2014
@@ -973,6 +973,22 @@ public interface IJobManager
public void noteConnectorRegistration(String[] connectionNames)
throws ManifoldCFException;
+ /** Note the deregistration of a notification connector used by the specified connections.
+ * This method will be called when the connector is deregistered. Jobs that use these connections
+ * must therefore enter appropriate states.
+ *@param connectionNames is the set of connection names.
+ */
+ public void noteNotificationConnectorDeregistration(String[] connectionNames)
+ throws ManifoldCFException;
+
+ /** Note the registration of a notification connector used by the specified connections.
+ * This method will be called when a connector is registered, on which the specified
+ * connections depend.
+ *@param connectionNames is the set of connection names.
+ */
+ public void noteNotificationConnectorRegistration(String[] connectionNames)
+ throws ManifoldCFException;
+
/** Note a change in connection configuration.
* This method will be called whenever a connection's configuration is modified, or when an external repository change
* is signalled.
Added: manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/NotificationConnectionManagerFactory.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/NotificationConnectionManagerFactory.java?rev=1645816&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/NotificationConnectionManagerFactory.java (added)
+++ manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/NotificationConnectionManagerFactory.java Tue Dec 16 02:00:08 2014
@@ -0,0 +1,58 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.interfaces;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+/** Notification connection manager factory.
+*/
+public class NotificationConnectionManagerFactory
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ // name to use in thread context pool of objects
+ private final static String objectName = "_NotificationConnectionMgr_";
+
+ private NotificationConnectionManagerFactory()
+ {
+ }
+
+ /** Make a notification connection manager handle.
+ *@param tc is the thread context.
+ *@return the handle.
+ */
+ public static INotificationConnectionManager make(IThreadContext tc)
+ throws ManifoldCFException
+ {
+ Object o = tc.get(objectName);
+ if (o == null || !(o instanceof INotificationConnectionManager))
+ {
+ IDBInterface database = DBInterfaceFactory.make(tc,
+ ManifoldCF.getMasterDatabaseName(),
+ ManifoldCF.getMasterDatabaseUsername(),
+ ManifoldCF.getMasterDatabasePassword());
+
+ o = new org.apache.manifoldcf.crawler.notification.NotificationConnectionManager(tc,database);
+ tc.save(objectName,o);
+ }
+ return (INotificationConnectionManager)o;
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/NotificationConnectionManagerFactory.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/NotificationConnectionManagerFactory.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified: manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1645816&r1=1645815&r2=1645816&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Tue Dec 16 02:00:08 2014
@@ -388,6 +388,116 @@ public class JobManager implements IJobM
}
}
+ /** Note the deregistration of a notification connector used by the specified connections.
+ * This method will be called when the connector is deregistered. Jobs that use these connections
+ * must therefore enter appropriate states.
+ *@param connectionNames is the set of connection names.
+ */
+ @Override
+ public void noteNotificationConnectorDeregistration(String[] connectionNames)
+ throws ManifoldCFException
+ {
+ // For each connection, find the corresponding list of jobs. From these jobs, we want the job id and the status.
+ List<String> list = new ArrayList<String>();
+ int maxCount = database.findConjunctionClauseMax(new ClauseDescription[]{});
+ int currentCount = 0;
+ int i = 0;
+ while (i < connectionNames.length)
+ {
+ if (currentCount == maxCount)
+ {
+ noteNotificationConnectionDeregistration(list);
+ list.clear();
+ currentCount = 0;
+ }
+
+ list.add(connectionNames[i++]);
+ currentCount++;
+ }
+ if (currentCount > 0)
+ noteNotificationConnectionDeregistration(list);
+ }
+
+ /** Note deregistration for a batch of notification connection names.
+ */
+ protected void noteNotificationConnectionDeregistration(List<String> list)
+ throws ManifoldCFException
+ {
+ // MHL
+ /*
+ ArrayList newList = new ArrayList();
+ String query = database.buildConjunctionClause(newList,new ClauseDescription[]{
+ new MultiClause(jobs.connectionNameField,list)});
+ // Query for the matching jobs, and then for each job potentially adjust the state
+ IResultSet set = database.performQuery("SELECT "+jobs.idField+","+jobs.statusField+" FROM "+
+ jobs.getTableName()+" WHERE "+query+" FOR UPDATE",
+ newList,null,null);
+ int i = 0;
+ while (i < set.getRowCount())
+ {
+ IResultRow row = set.getRow(i++);
+ Long jobID = (Long)row.getValue(jobs.idField);
+ int statusValue = jobs.stringToStatus((String)row.getValue(jobs.statusField));
+ jobs.noteConnectorDeregistration(jobID,statusValue);
+ }
+ */
+ }
+
+ /** Note the registration of a notification connector used by the specified connections.
+ * This method will be called when a connector is registered, on which the specified
+ * connections depend.
+ *@param connectionNames is the set of connection names.
+ */
+ @Override
+ public void noteNotificationConnectorRegistration(String[] connectionNames)
+ throws ManifoldCFException
+ {
+ // For each connection, find the corresponding list of jobs. From these jobs, we want the job id and the status.
+ List<String> list = new ArrayList<String>();
+ int maxCount = database.findConjunctionClauseMax(new ClauseDescription[]{});
+ int currentCount = 0;
+ int i = 0;
+ while (i < connectionNames.length)
+ {
+ if (currentCount == maxCount)
+ {
+ noteNotificationConnectionRegistration(list);
+ list.clear();
+ currentCount = 0;
+ }
+
+ list.add(connectionNames[i++]);
+ currentCount++;
+ }
+ if (currentCount > 0)
+ noteNotificationConnectionRegistration(list);
+ }
+
+ /** Note registration for a batch of connection names.
+ */
+ protected void noteNotificationConnectionRegistration(List<String> list)
+ throws ManifoldCFException
+ {
+ // Query for the matching jobs, and then for each job potentially adjust the state
+ // MHL
+ /*
+ ArrayList newList = new ArrayList();
+ String query = database.buildConjunctionClause(newList,new ClauseDescription[]{
+ new MultiClause(jobs.connectionNameField,list)});
+ IResultSet set = database.performQuery("SELECT "+jobs.idField+","+jobs.statusField+" FROM "+
+ jobs.getTableName()+" WHERE "+query+" FOR UPDATE",
+ newList,null,null);
+ int i = 0;
+ while (i < set.getRowCount())
+ {
+ IResultRow row = set.getRow(i++);
+ Long jobID = (Long)row.getValue(jobs.idField);
+ int statusValue = jobs.stringToStatus((String)row.getValue(jobs.statusField));
+ jobs.noteConnectorRegistration(jobID,statusValue);
+ }
+ */
+ }
+
/** Note the deregistration of an output connector used by the specified connections.
* This method will be called when the connector is deregistered. Jobs that use these connections
* must therefore enter appropriate states.
Modified: manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1645816&r1=1645815&r2=1645816&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java (original)
+++ manifoldcf/branches/CONNECTORS-1119/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java Tue Dec 16 02:00:08 2014
@@ -198,6 +198,7 @@ public class ManifoldCF extends org.apac
protected static final String NODE_TRANSFORMATIONCONNECTOR = "transformationconnector";
protected static final String NODE_MAPPINGCONNECTOR = "mappingconnector";
protected static final String NODE_AUTHORITYCONNECTOR = "authorityconnector";
+ protected static final String NODE_NOTIFICATIONCONNECTOR = "notificationconnector";
protected static final String NODE_REPOSITORYCONNECTOR = "repositoryconnector";
protected static final String ATTRIBUTE_NAME = "name";
protected static final String ATTRIBUTE_CLASS = "class";
@@ -214,6 +215,7 @@ public class ManifoldCF extends org.apac
Map<String,String> desiredTransformationConnectors = new HashMap<String,String>();
Map<String,String> desiredMappingConnectors = new HashMap<String,String>();
Map<String,String> desiredAuthorityConnectors = new HashMap<String,String>();
+ Map<String,String> desiredNotificationConnectors = new HashMap<String,String>();
Map<String,String> desiredRepositoryConnectors = new HashMap<String,String>();
Map<String,String> desiredDomains = new HashMap<String,String>();
@@ -253,6 +255,12 @@ public class ManifoldCF extends org.apac
String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
desiredAuthorityConnectors.put(className,name);
}
+ else if (cn.getType().equals(NODE_NOTIFICATIONCONNECTOR))
+ {
+ String name = cn.getAttributeValue(ATTRIBUTE_NAME);
+ String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
+ desiredNotificationConnectors.put(className,name);
+ }
else if (cn.getType().equals(NODE_REPOSITORYCONNECTOR))
{
String name = cn.getAttributeValue(ATTRIBUTE_NAME);
@@ -407,6 +415,49 @@ public class ManifoldCF extends org.apac
System.err.println("Successfully unregistered all authority connectors");
}
+ // Notification connectors...
+ {
+ INotificationConnectorManager mgr = NotificationConnectorManagerFactory.make(tc);
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ INotificationConnectionManager connManager = NotificationConnectionManagerFactory.make(tc);
+ IResultSet classNames = mgr.getConnectors();
+ int i = 0;
+ while (i < classNames.getRowCount())
+ {
+ IResultRow row = classNames.getRow(i++);
+ String className = (String)row.getValue("classname");
+ String description = (String)row.getValue("description");
+ if (desiredNotificationConnectors.get(className) == null || !desiredNotificationConnectors.get(className).equals(description))
+ {
+ // Deregistration should be done in a transaction
+ database.beginTransaction();
+ try
+ {
+ // Find the connection names that come with this class
+ String[] connectionNames = connManager.findConnectionsForConnector(className);
+ // For each connection name, modify the jobs to note that the connector is no longer installed
+ jobManager.noteNotificationConnectorDeregistration(connectionNames);
+ // Now that all jobs have been placed into an appropriate state, actually do the deregistration itself.
+ mgr.unregisterConnector(className);
+ }
+ catch (ManifoldCFException e)
+ {
+ database.signalRollback();
+ throw e;
+ }
+ catch (Error e)
+ {
+ database.signalRollback();
+ throw e;
+ }
+ finally
+ {
+ database.endTransaction();
+ }
+ }
+ }
+ }
+
// Repository connectors...
{
IConnectorManager mgr = ConnectorManagerFactory.make(tc);
@@ -562,6 +613,41 @@ public class ManifoldCF extends org.apac
mgr.registerConnector(name,className);
System.err.println("Successfully registered mapping connector '"+className+"'");
}
+ else if (cn.getType().equals(NODE_NOTIFICATIONCONNECTOR))
+ {
+ String name = cn.getAttributeValue(ATTRIBUTE_NAME);
+ String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
+ INotificationConnectorManager mgr = NotificationConnectorManagerFactory.make(tc);
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ INotificationConnectionManager connManager = NotificationConnectionManagerFactory.make(tc);
+ // Deregistration should be done in a transaction
+ database.beginTransaction();
+ try
+ {
+ // First, register connector
+ mgr.registerConnector(name,className);
+ // Then, signal to all jobs that might depend on this connector that they can switch state
+ // Find the connection names that come with this class
+ String[] connectionNames = connManager.findConnectionsForConnector(className);
+ // For each connection name, modify the jobs to note that the connector is now installed
+ jobManager.noteNotificationConnectorRegistration(connectionNames);
+ }
+ catch (ManifoldCFException e)
+ {
+ database.signalRollback();
+ throw e;
+ }
+ catch (Error e)
+ {
+ database.signalRollback();
+ throw e;
+ }
+ finally
+ {
+ database.endTransaction();
+ }
+ System.err.println("Successfully registered notification connector '"+className+"'");
+ }
else if (cn.getType().equals(NODE_REPOSITORYCONNECTOR))
{
String name = cn.getAttributeValue(ATTRIBUTE_NAME);
@@ -611,11 +697,15 @@ public class ManifoldCF extends org.apac
{
IConnectorManager repConnMgr = ConnectorManagerFactory.make(threadcontext);
IRepositoryConnectionManager repCon = RepositoryConnectionManagerFactory.make(threadcontext);
+ INotificationConnectorManager notConnMgr = NotificationConnectorManagerFactory.make(threadcontext);
+ INotificationConnectionManager notCon = NotificationConnectionManagerFactory.make(threadcontext);
IJobManager jobManager = JobManagerFactory.make(threadcontext);
IBinManager binManager = BinManagerFactory.make(threadcontext);
org.apache.manifoldcf.authorities.system.ManifoldCF.installSystemTables(threadcontext);
repConnMgr.install();
repCon.install();
+ notConnMgr.install();
+ notCon.install();
jobManager.install();
binManager.install();
}
@@ -628,10 +718,14 @@ public class ManifoldCF extends org.apac
{
IConnectorManager repConnMgr = ConnectorManagerFactory.make(threadcontext);
IRepositoryConnectionManager repCon = RepositoryConnectionManagerFactory.make(threadcontext);
+ INotificationConnectorManager notConnMgr = NotificationConnectorManagerFactory.make(threadcontext);
+ INotificationConnectionManager notCon = NotificationConnectionManagerFactory.make(threadcontext);
IJobManager jobManager = JobManagerFactory.make(threadcontext);
IBinManager binManager = BinManagerFactory.make(threadcontext);
binManager.deinstall();
jobManager.deinstall();
+ notCon.deinstall();
+ notConnMgr.deinstall();
repCon.deinstall();
repConnMgr.deinstall();
org.apache.manifoldcf.authorities.system.ManifoldCF.deinstallSystemTables(threadcontext);
@@ -652,6 +746,7 @@ public class ManifoldCF extends org.apac
ITransformationConnectionManager transManager = TransformationConnectionManagerFactory.make(threadContext);
IAuthorityGroupManager groupManager = AuthorityGroupManagerFactory.make(threadContext);
IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(threadContext);
+ INotificationConnectionManager notificationConnManager = NotificationConnectionManagerFactory.make(threadContext);
IMappingConnectionManager mappingManager = MappingConnectionManagerFactory.make(threadContext);
IAuthorityConnectionManager authManager = AuthorityConnectionManagerFactory.make(threadContext);
IJobManager jobManager = JobManagerFactory.make(threadContext);
@@ -728,6 +823,11 @@ public class ManifoldCF extends org.apac
connManager.exportConfiguration(zos);
zos.closeEntry();
+ java.util.zip.ZipEntry notConnEntry = new java.util.zip.ZipEntry("notifications");
+ zos.putNextEntry(notConnEntry);
+ notificationConnManager.exportConfiguration(zos);
+ zos.closeEntry();
+
java.util.zip.ZipEntry jobsEntry = new java.util.zip.ZipEntry("jobs");
zos.putNextEntry(jobsEntry);
jobManager.exportConfiguration(zos);
@@ -787,6 +887,7 @@ public class ManifoldCF extends org.apac
ITransformationConnectionManager transManager = TransformationConnectionManagerFactory.make(threadContext);
IAuthorityGroupManager groupManager = AuthorityGroupManagerFactory.make(threadContext);
IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(threadContext);
+ INotificationConnectionManager notificationConnManager = NotificationConnectionManagerFactory.make(threadContext);
IMappingConnectionManager mappingManager = MappingConnectionManagerFactory.make(threadContext);
IAuthorityConnectionManager authManager = AuthorityConnectionManagerFactory.make(threadContext);
IJobManager jobManager = JobManagerFactory.make(threadContext);
@@ -845,6 +946,8 @@ public class ManifoldCF extends org.apac
authManager.importConfiguration(zis);
else if (name.equals("connections"))
connManager.importConfiguration(zis);
+ else if (name.equals("notifications"))
+ notificationConnManager.importConfiguration(zis);
else if (name.equals("jobs"))
jobManager.importConfiguration(zis);
else