You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/06/10 01:19:10 UTC
svn commit: r1601529 [5/6] - in /manifoldcf/trunk: ./ connectors/
connectors/nulltransformation/
framework/agents/src/main/java/org/apache/manifoldcf/agents/
framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/
framework/agent...
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java?rev=1601529&r1=1601528&r2=1601529&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java Mon Jun 9 23:19:08 2014
@@ -50,7 +50,8 @@ public class CrawlerAgent implements IAg
protected IdleCleanupThread idleCleanupThread = null;
protected SetPriorityThread setPriorityThread = null;
protected HistoryCleanupThread historyCleanupThread = null;
-
+ protected AssessmentThread assessmentThread = null;
+
// Reset managers
/** Worker thread pool reset manager */
protected WorkerResetManager workerResetManager = null;
@@ -282,6 +283,55 @@ public class CrawlerAgent implements IAg
jobManager.noteOutputConnectionChange(connectionName);
}
+ /** Request permission from agent to delete a transformation connection.
+ *@param connName is the name of the transformation connection.
+ *@return true if the connection is in use, false otherwise.
+ */
+ @Override
+ public boolean isTransformationConnectionInUse(IThreadContext threadContext, String connName)
+ throws ManifoldCFException
+ {
+ // Check with job manager.
+ IJobManager jobManager = JobManagerFactory.make(threadContext);
+ return jobManager.checkIfTransformationReference(connName);
+ }
+
+ /** Note the deregistration of a set of transformation connections.
+ *@param connectionNames are the names of the connections being deregistered.
+ */
+ @Override
+ public void noteTransformationConnectorDeregistration(IThreadContext threadContext, String[] connectionNames)
+ throws ManifoldCFException
+ {
+ // Notify job manager
+ IJobManager jobManager = JobManagerFactory.make(threadContext);
+ jobManager.noteTransformationConnectorDeregistration(connectionNames);
+ }
+
+ /** Note the registration of a set of transformation connections.
+ *@param connectionNames are the names of the connections being registered.
+ */
+ @Override
+ public void noteTransformationConnectorRegistration(IThreadContext threadContext, String[] connectionNames)
+ throws ManifoldCFException
+ {
+ // Notify job manager
+ IJobManager jobManager = JobManagerFactory.make(threadContext);
+ jobManager.noteTransformationConnectorRegistration(connectionNames);
+ }
+
+ /** Note a change in configuration for a transformation connection.
+ *@param connectionName is the name of the connection being changed.
+ */
+ @Override
+ public void noteTransformationConnectionChange(IThreadContext threadContext, String connectionName)
+ throws ManifoldCFException
+ {
+ // Notify job manager
+ IJobManager jobManager = JobManagerFactory.make(threadContext);
+ jobManager.noteTransformationConnectionChange(connectionName);
+ }
+
/** Start everything.
*/
public void startSystem(IThreadContext threadContext)
@@ -373,6 +423,7 @@ public class CrawlerAgent implements IAg
jobResetThread = new JobResetThread(processID);
seedingThread = new SeedingThread(new SeedingResetManager(processID),processID);
idleCleanupThread = new IdleCleanupThread(processID);
+ assessmentThread = new AssessmentThread(processID);
// Start all the threads
jobStartThread.start();
@@ -419,6 +470,7 @@ public class CrawlerAgent implements IAg
jobResetThread.start();
seedingThread.start();
idleCleanupThread.start();
+ assessmentThread.start();
Logging.root.info("Pull-agent started");
}
@@ -434,7 +486,7 @@ public class CrawlerAgent implements IAg
finisherThread != null || notificationThread != null || workerThreads != null || expireStufferThread != null || expireThreads != null ||
deleteStufferThread != null || deleteThreads != null ||
cleanupStufferThread != null || cleanupThreads != null ||
- jobResetThread != null || seedingThread != null || idleCleanupThread != null || setPriorityThread != null || historyCleanupThread != null)
+ jobResetThread != null || seedingThread != null || idleCleanupThread != null || assessmentThread != null || setPriorityThread != null || historyCleanupThread != null)
{
// Send an interrupt to all threads that are still there.
// In theory, this only needs to be done once. In practice, I have seen cases where the thread loses track of the fact that it has been
@@ -539,6 +591,10 @@ public class CrawlerAgent implements IAg
{
idleCleanupThread.interrupt();
}
+ if (assessmentThread != null)
+ {
+ assessmentThread.interrupt();
+ }
// Now, wait for all threads to die.
try
@@ -704,6 +760,11 @@ public class CrawlerAgent implements IAg
if (!idleCleanupThread.isAlive())
idleCleanupThread = null;
}
+ if (assessmentThread != null)
+ {
+ if (!assessmentThread.isAlive())
+ assessmentThread = null;
+ }
}
// Threads are down; release connectors
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1601529&r1=1601528&r2=1601529&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java Mon Jun 9 23:19:08 2014
@@ -196,6 +196,7 @@ public class ManifoldCF extends org.apac
// Connectors configuration file
protected static final String NODE_AUTHORIZATIONDOMAIN = "authorizationdomain";
protected static final String NODE_OUTPUTCONNECTOR = "outputconnector";
+ protected static final String NODE_TRANSFORMATIONCONNECTOR = "transformationconnector";
protected static final String NODE_MAPPINGCONNECTOR = "mappingconnector";
protected static final String NODE_AUTHORITYCONNECTOR = "authorityconnector";
protected static final String NODE_REPOSITORYCONNECTOR = "repositoryconnector";
@@ -211,6 +212,7 @@ public class ManifoldCF extends org.apac
// Create a map of class name and description, so we can compare what we can find
// against what we want.
Map<String,String> desiredOutputConnectors = new HashMap<String,String>();
+ Map<String,String> desiredTransformationConnectors = new HashMap<String,String>();
Map<String,String> desiredMappingConnectors = new HashMap<String,String>();
Map<String,String> desiredAuthorityConnectors = new HashMap<String,String>();
Map<String,String> desiredRepositoryConnectors = new HashMap<String,String>();
@@ -234,6 +236,12 @@ public class ManifoldCF extends org.apac
String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
desiredOutputConnectors.put(className,name);
}
+ else if (cn.getType().equals(NODE_TRANSFORMATIONCONNECTOR))
+ {
+ String name = cn.getAttributeValue(ATTRIBUTE_NAME);
+ String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
+ desiredTransformationConnectors.put(className,name);
+ }
else if (cn.getType().equals(NODE_MAPPINGCONNECTOR))
{
String name = cn.getAttributeValue(ATTRIBUTE_NAME);
@@ -321,6 +329,49 @@ public class ManifoldCF extends org.apac
System.err.println("Successfully unregistered all output connectors");
}
+ // Output connectors...
+ {
+ ITransformationConnectorManager mgr = TransformationConnectorManagerFactory.make(tc);
+ ITransformationConnectionManager connManager = TransformationConnectionManagerFactory.make(tc);
+ IResultSet classNames = mgr.getConnectors();
+ int i = 0;
+ while (i < classNames.getRowCount())
+ {
+ IResultRow row = classNames.getRow(i++);
+ String className = (String)row.getValue("classname");
+ String description = (String)row.getValue("description");
+ if (desiredTransformationConnectors.get(className) == null || !desiredTransformationConnectors.get(className).equals(description))
+ {
+ // Deregistration should be done in a transaction
+ database.beginTransaction();
+ try
+ {
+ // Find the connection names that come with this class
+ String[] connectionNames = connManager.findConnectionsForConnector(className);
+ // For all connection names, notify all agents of the deregistration
+ AgentManagerFactory.noteTransformationConnectorDeregistration(tc,connectionNames);
+ // Now that all jobs have been placed into an appropriate state, actually do the deregistration itself.
+ mgr.unregisterConnector(className);
+ }
+ catch (ManifoldCFException e)
+ {
+ database.signalRollback();
+ throw e;
+ }
+ catch (Error e)
+ {
+ database.signalRollback();
+ throw e;
+ }
+ finally
+ {
+ database.endTransaction();
+ }
+ }
+ }
+ System.err.println("Successfully unregistered all transformation connectors");
+ }
+
// Mapping connectors...
{
IMappingConnectorManager mgr = MappingConnectorManagerFactory.make(tc);
@@ -462,6 +513,40 @@ public class ManifoldCF extends org.apac
}
System.err.println("Successfully registered output connector '"+className+"'");
}
+ else if (cn.getType().equals(NODE_TRANSFORMATIONCONNECTOR))
+ {
+ String name = cn.getAttributeValue(ATTRIBUTE_NAME);
+ String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
+ ITransformationConnectorManager mgr = TransformationConnectorManagerFactory.make(tc);
+ ITransformationConnectionManager connManager = TransformationConnectionManagerFactory.make(tc);
+ // Registration should be done in a transaction
+ database.beginTransaction();
+ try
+ {
+ // First, register connector
+ mgr.registerConnector(name,className);
+ // Then, signal to all jobs that might depend on this connector that they can switch state
+ // Find the connection names that come with this class
+ String[] connectionNames = connManager.findConnectionsForConnector(className);
+ // For all connection names, notify all agents of the registration
+ AgentManagerFactory.noteTransformationConnectorRegistration(tc,connectionNames);
+ }
+ catch (ManifoldCFException e)
+ {
+ database.signalRollback();
+ throw e;
+ }
+ catch (Error e)
+ {
+ database.signalRollback();
+ throw e;
+ }
+ finally
+ {
+ database.endTransaction();
+ }
+ System.err.println("Successfully registered transformation connector '"+className+"'");
+ }
else if (cn.getType().equals(NODE_AUTHORITYCONNECTOR))
{
String name = cn.getAttributeValue(ATTRIBUTE_NAME);
@@ -565,6 +650,7 @@ public class ManifoldCF extends org.apac
ManifoldCF.getMasterDatabasePassword());
// Also create the following managers, which will handle the actual details of writing configuration data
IOutputConnectionManager outputManager = OutputConnectionManagerFactory.make(threadContext);
+ ITransformationConnectionManager transManager = TransformationConnectionManagerFactory.make(threadContext);
IAuthorityGroupManager groupManager = AuthorityGroupManagerFactory.make(threadContext);
IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(threadContext);
IMappingConnectionManager mappingManager = MappingConnectionManagerFactory.make(threadContext);
@@ -624,6 +710,11 @@ public class ManifoldCF extends org.apac
// The zipentries must be written in an order that permits their proper restoration. The "lowest level" is thus
// written first, which yields the order: authority connections, repository connections, jobs
+ java.util.zip.ZipEntry transEntry = new java.util.zip.ZipEntry("transformations");
+ zos.putNextEntry(transEntry);
+ transManager.exportConfiguration(zos);
+ zos.closeEntry();
+
java.util.zip.ZipEntry outputEntry = new java.util.zip.ZipEntry("outputs");
zos.putNextEntry(outputEntry);
outputManager.exportConfiguration(zos);
@@ -705,6 +796,7 @@ public class ManifoldCF extends org.apac
ManifoldCF.getMasterDatabasePassword());
// Also create the following managers, which will handle the actual details of reading configuration data
IOutputConnectionManager outputManager = OutputConnectionManagerFactory.make(threadContext);
+ ITransformationConnectionManager transManager = TransformationConnectionManagerFactory.make(threadContext);
IAuthorityGroupManager groupManager = AuthorityGroupManagerFactory.make(threadContext);
IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(threadContext);
IMappingConnectionManager mappingManager = MappingConnectionManagerFactory.make(threadContext);
@@ -761,7 +853,9 @@ public class ManifoldCF extends org.apac
entries++;
// Get the name of the entry
String name = z.getName();
- if (name.equals("outputs"))
+ if (name.equals("transformations"))
+ transManager.importConfiguration(zis);
+ else if (name.equals("outputs"))
outputManager.importConfiguration(zis);
else if (name.equals("groups"))
groupManager.importConfiguration(zis);
@@ -1029,16 +1123,6 @@ public class ManifoldCF extends org.apac
}
- /** Qualify output activity name.
- *@param outputActivityName is the name of the output activity.
- *@param outputConnectionName is the corresponding name of the output connection.
- *@return the qualified (global) activity name.
- */
- public static String qualifyOutputActivityName(String outputActivityName, String outputConnectionName)
- {
- return outputActivityName+" ("+outputConnectionName+")";
- }
-
/** Get the activities list for a given repository connection.
*/
public static String[] getActivitiesList(IThreadContext threadContext, String connectionName)
@@ -1049,31 +1133,35 @@ public class ManifoldCF extends org.apac
if (thisConnection == null)
return null;
String[] outputActivityList = OutputConnectionManagerFactory.getAllOutputActivities(threadContext);
+ String[] transformationActivityList = TransformationConnectionManagerFactory.getAllTransformationActivities(threadContext);
String[] connectorActivityList = RepositoryConnectorFactory.getActivitiesList(threadContext,thisConnection.getClassName());
String[] globalActivityList = IRepositoryConnectionManager.activitySet;
- String[] activityList = new String[outputActivityList.length + ((connectorActivityList==null)?0:connectorActivityList.length) + globalActivityList.length];
+ String[] activityList = new String[transformationActivityList.length + outputActivityList.length + ((connectorActivityList==null)?0:connectorActivityList.length) + globalActivityList.length];
int k2 = 0;
- int j;
+ if (transformationActivityList != null)
+ {
+ for (String transformationActivity: transformationActivityList)
+ {
+ activityList[k2++] = transformationActivity;
+ }
+ }
if (outputActivityList != null)
{
- j = 0;
- while (j < outputActivityList.length)
+ for (String outputActivity : outputActivityList)
{
- activityList[k2++] = outputActivityList[j++];
+ activityList[k2++] = outputActivity;
}
}
if (connectorActivityList != null)
{
- j = 0;
- while (j < connectorActivityList.length)
+ for (String connectorActivity : connectorActivityList)
{
- activityList[k2++] = connectorActivityList[j++];
+ activityList[k2++] = connectorActivity;
}
}
- j = 0;
- while (j < globalActivityList.length)
+ for (String globalActivity : globalActivityList)
{
- activityList[k2++] = globalActivityList[j++];
+ activityList[k2++] = globalActivity;
}
java.util.Arrays.sort(activityList);
return activityList;
@@ -1109,10 +1197,12 @@ public class ManifoldCF extends org.apac
protected static final String API_AUTHORITYGROUPNODE = "authoritygroup";
protected static final String API_REPOSITORYCONNECTORNODE = "repositoryconnector";
protected static final String API_OUTPUTCONNECTORNODE = "outputconnector";
+ protected static final String API_TRANSFORMATIONCONNECTORNODE = "transformationconnector";
protected static final String API_AUTHORITYCONNECTORNODE = "authorityconnector";
protected static final String API_MAPPINGCONNECTORNODE = "mappingconnector";
protected static final String API_REPOSITORYCONNECTIONNODE = "repositoryconnection";
protected static final String API_OUTPUTCONNECTIONNODE = "outputconnection";
+ protected static final String API_TRANSFORMATIONCONNECTIONNODE = "transformationconnection";
protected static final String API_AUTHORITYCONNECTIONNODE = "authorityconnection";
protected static final String API_MAPPINGCONNECTIONNODE = "mappingconnection";
protected static final String API_CHECKRESULTNODE = "check_result";
@@ -1262,6 +1352,48 @@ public class ManifoldCF extends org.apac
return READRESULT_FOUND;
}
+ /** Read a transformation connection status */
+ protected static int apiReadTransformationConnectionStatus(IThreadContext tc, Configuration output, String connectionName)
+ throws ManifoldCFException
+ {
+ try
+ {
+ ITransformationConnectorPool transformationConnectorPool = TransformationConnectorPoolFactory.make(tc);
+ ITransformationConnectionManager connectionManager = TransformationConnectionManagerFactory.make(tc);
+ ITransformationConnection connection = connectionManager.load(connectionName);
+ if (connection == null)
+ {
+ createErrorNode(output,"Connection '"+connectionName+"' does not exist");
+ return READRESULT_NOTFOUND;
+ }
+
+ String results;
+ // Grab a connection handle, and call the test method
+ ITransformationConnector connector = transformationConnectorPool.grab(connection);
+ try
+ {
+ results = connector.check();
+ }
+ catch (ManifoldCFException e)
+ {
+ results = e.getMessage();
+ }
+ finally
+ {
+ transformationConnectorPool.release(connection,connector);
+ }
+
+ ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE);
+ response.setValue(results);
+ output.addChild(output.getChildCount(),response);
+ }
+ catch (ManifoldCFException e)
+ {
+ createErrorNode(output,e);
+ }
+ return READRESULT_FOUND;
+ }
+
/** Read an authority connection status */
protected static int apiReadAuthorityConnectionStatus(IThreadContext tc, Configuration output, String connectionName)
throws ManifoldCFException
@@ -1421,6 +1553,39 @@ public class ManifoldCF extends org.apac
}
return READRESULT_FOUND;
}
+
+ /** Read a transformation connection's info */
+ protected static int apiReadTransformationConnectionInfo(IThreadContext tc, Configuration output, String connectionName, String command)
+ throws ManifoldCFException
+ {
+ try
+ {
+ ITransformationConnectorPool transformationConnectorPool = TransformationConnectorPoolFactory.make(tc);
+ ITransformationConnectionManager connectionManager = TransformationConnectionManagerFactory.make(tc);
+ ITransformationConnection connection = connectionManager.load(connectionName);
+ if (connection == null)
+ {
+ createErrorNode(output,"Connection '"+connectionName+"' does not exist");
+ return READRESULT_NOTFOUND;
+ }
+
+ // Grab a connection handle, and call the test method
+ ITransformationConnector connector = transformationConnectorPool.grab(connection);
+ try
+ {
+ return connector.requestInfo(output,command)?READRESULT_FOUND:READRESULT_NOTFOUND;
+ }
+ finally
+ {
+ transformationConnectorPool.release(connection,connector);
+ }
+ }
+ catch (ManifoldCFException e)
+ {
+ createErrorNode(output,e);
+ }
+ return READRESULT_FOUND;
+ }
/** Read a repository connection's info */
protected static int apiReadRepositoryConnectionInfo(IThreadContext tc, Configuration output, String connectionName, String command)
@@ -1669,6 +1834,57 @@ public class ManifoldCF extends org.apac
return READRESULT_FOUND;
}
+ /** Get transformation connections */
+ protected static int apiReadTransformationConnections(IThreadContext tc, Configuration output)
+ throws ManifoldCFException
+ {
+ try
+ {
+ ITransformationConnectionManager connManager = TransformationConnectionManagerFactory.make(tc);
+ ITransformationConnection[] connections = connManager.getAllConnections();
+ int i = 0;
+ while (i < connections.length)
+ {
+ ConfigurationNode connectionNode = new ConfigurationNode(API_TRANSFORMATIONCONNECTIONNODE);
+ formatTransformationConnection(connectionNode,connections[i++]);
+ output.addChild(output.getChildCount(),connectionNode);
+ }
+ }
+ catch (ManifoldCFException e)
+ {
+ createErrorNode(output,e);
+ }
+ return READRESULT_FOUND;
+ }
+
+ /** Read transformation connection */
+ protected static int apiReadTransformationConnection(IThreadContext tc, Configuration output, String connectionName)
+ throws ManifoldCFException
+ {
+ try
+ {
+ ITransformationConnectionManager connectionManager = TransformationConnectionManagerFactory.make(tc);
+ ITransformationConnection connection = connectionManager.load(connectionName);
+ if (connection != null)
+ {
+ // Fill the return object with job information
+ ConfigurationNode connectionNode = new ConfigurationNode(API_TRANSFORMATIONCONNECTIONNODE);
+ formatTransformationConnection(connectionNode,connection);
+ output.addChild(output.getChildCount(),connectionNode);
+ }
+ else
+ {
+ createErrorNode(output,"Connection '"+connectionName+"' does not exist.");
+ return READRESULT_NOTFOUND;
+ }
+ }
+ catch (ManifoldCFException e)
+ {
+ createErrorNode(output,e);
+ }
+ return READRESULT_FOUND;
+ }
+
/** Get authority connections */
protected static int apiReadAuthorityConnections(IThreadContext tc, Configuration output)
throws ManifoldCFException
@@ -1859,6 +2075,43 @@ public class ManifoldCF extends org.apac
return READRESULT_FOUND;
}
+ /** List transformation connectors */
+ protected static int apiReadTransformationConnectors(IThreadContext tc, Configuration output)
+ throws ManifoldCFException
+ {
+ // List registered transformation connectors
+ try
+ {
+ ITransformationConnectorManager manager = TransformationConnectorManagerFactory.make(tc);
+ IResultSet resultSet = manager.getConnectors();
+ int j = 0;
+ while (j < resultSet.getRowCount())
+ {
+ IResultRow row = resultSet.getRow(j++);
+ ConfigurationNode child = new ConfigurationNode(API_TRANSFORMATIONCONNECTORNODE);
+ String description = (String)row.getValue("description");
+ String className = (String)row.getValue("classname");
+ ConfigurationNode node;
+ if (description != null)
+ {
+ node = new ConfigurationNode(CONNECTORNODE_DESCRIPTION);
+ node.setValue(description);
+ child.addChild(child.getChildCount(),node);
+ }
+ node = new ConfigurationNode(CONNECTORNODE_CLASSNAME);
+ node.setValue(className);
+ child.addChild(child.getChildCount(),node);
+
+ output.addChild(output.getChildCount(),child);
+ }
+ }
+ catch (ManifoldCFException e)
+ {
+ createErrorNode(output,e);
+ }
+ return READRESULT_FOUND;
+ }
+
/** List authority connectors */
protected static int apiReadAuthorityConnectors(IThreadContext tc, Configuration output)
throws ManifoldCFException
@@ -2681,6 +2934,10 @@ public class ManifoldCF extends org.apac
{
return apiReadOutputConnectionStatus(tc,output,connectionName);
}
+ else if (connectionType.equals("transformationconnections"))
+ {
+ return apiReadTransformationConnectionStatus(tc,output,connectionName);
+ }
else if (connectionType.equals("mappingconnections"))
{
return apiReadMappingConnectionStatus(tc,output,connectionName);
@@ -2724,6 +2981,10 @@ public class ManifoldCF extends org.apac
{
return apiReadOutputConnectionInfo(tc,output,connectionName,command);
}
+ else if (connectionType.equals("transformationconnections"))
+ {
+ return apiReadTransformationConnectionInfo(tc,output,connectionName,command);
+ }
else if (connectionType.equals("repositoryconnections"))
{
return apiReadRepositoryConnectionInfo(tc,output,connectionName,command);
@@ -2770,6 +3031,15 @@ public class ManifoldCF extends org.apac
String connectionName = decodeAPIPathElement(path.substring("outputconnections/".length()));
return apiReadOutputConnection(tc,output,connectionName);
}
+ else if (path.equals("transformationconnections"))
+ {
+ return apiReadTransformationConnections(tc,output);
+ }
+ else if (path.startsWith("transformationconnections/"))
+ {
+ String connectionName = decodeAPIPathElement(path.substring("transformationconnections/".length()));
+ return apiReadTransformationConnection(tc,output,connectionName);
+ }
else if (path.equals("mappingconnections"))
{
return apiReadMappingConnections(tc,output);
@@ -2801,6 +3071,10 @@ public class ManifoldCF extends org.apac
{
return apiReadOutputConnectors(tc,output);
}
+ else if (path.equals("transformationconnectors"))
+ {
+ return apiReadTransformationConnectors(tc,output);
+ }
else if (path.equals("mappingconnectors"))
{
return apiReadMappingConnectors(tc,output);
@@ -3109,6 +3383,41 @@ public class ManifoldCF extends org.apac
}
return WRITERESULT_FOUND;
}
+
+ /** Write transformation connection.
+ */
+ protected static int apiWriteTransformationConnection(IThreadContext tc, Configuration output, Configuration input, String connectionName)
+ throws ManifoldCFException
+ {
+ ConfigurationNode connectionNode = findConfigurationNode(input,API_TRANSFORMATIONCONNECTIONNODE);
+ if (connectionNode == null)
+ throw new ManifoldCFException("Input argument must have '"+API_TRANSFORMATIONCONNECTIONNODE+"' field");
+
+ // Turn the configuration node into a TransformationConnection
+ org.apache.manifoldcf.agents.transformationconnection.TransformationConnection transformationConnection = new org.apache.manifoldcf.agents.transformationconnection.TransformationConnection();
+ processTransformationConnection(transformationConnection,connectionNode);
+
+ if (transformationConnection.getName() == null)
+ transformationConnection.setName(connectionName);
+ else
+ {
+ if (!transformationConnection.getName().equals(connectionName))
+ throw new ManifoldCFException("Connection name in path and in object must agree");
+ }
+
+ try
+ {
+ // Save the connection.
+ ITransformationConnectionManager connectionManager = TransformationConnectionManagerFactory.make(tc);
+ if (connectionManager.save(transformationConnection))
+ return WRITERESULT_CREATED;
+ }
+ catch (ManifoldCFException e)
+ {
+ createErrorNode(output,e);
+ }
+ return WRITERESULT_FOUND;
+ }
/** Write authority connection.
*/
@@ -3332,6 +3641,11 @@ public class ManifoldCF extends org.apac
String connectionName = decodeAPIPathElement(path.substring("outputconnections/".length()));
return apiWriteOutputConnection(tc,output,input,connectionName);
}
+ else if (path.startsWith("transformationconnections/"))
+ {
+ String connectionName = decodeAPIPathElement(path.substring("transformationconnections/".length()));
+ return apiWriteTransformationConnection(tc,output,input,connectionName);
+ }
else if (path.startsWith("mappingconnections/"))
{
String connectionName = decodeAPIPathElement(path.substring("mappingconnections/".length()));
@@ -3560,6 +3874,10 @@ public class ManifoldCF extends org.apac
protected static final String JOBNODE_FORCEDPARAM = "forcedparam";
protected static final String JOBNODE_PARAMNAME = "paramname";
protected static final String JOBNODE_PARAMVALUE = "paramvalue";
+ protected static final String JOBNODE_PIPELINESTAGE = "pipelinestage";
+ protected static final String JOBNODE_STAGECONNECTIONNAME = "stage_connectionname";
+ protected static final String JOBNODE_STAGEDESCRIPTION = "stage_description";
+ protected static final String JOBNODE_STAGESPECIFICATION = "stage_specification";
/** Convert a node into a job description.
*@param jobDescription is the job to be filled in.
@@ -3592,15 +3910,43 @@ public class ManifoldCF extends org.apac
{
jobDescription.setOutputConnectionName(child.getValue());
}
+ else if (childType.equals(JOBNODE_PIPELINESTAGE))
+ {
+ String stageConnectionName = null;
+ String stageDescription = null;
+ ConfigurationNode stageSpecification = null;
+ for (int q = 0; q < child.getChildCount(); q++)
+ {
+ ConfigurationNode cn = child.findChild(q);
+ if (cn.getType().equals(JOBNODE_STAGECONNECTIONNAME))
+ stageConnectionName = cn.getValue();
+ else if (cn.getType().equals(JOBNODE_STAGEDESCRIPTION))
+ stageDescription = cn.getValue();
+ else if (cn.getType().equals(JOBNODE_STAGESPECIFICATION))
+ {
+ stageSpecification = cn;
+ }
+ else
+ throw new ManifoldCFException("Found an unexpected node type: '"+cn.getType()+"'");
+ }
+ if (stageConnectionName == null)
+ throw new ManifoldCFException("Missing required field: '"+JOBNODE_STAGECONNECTIONNAME+"'");
+ OutputSpecification os = jobDescription.addPipelineStage(stageConnectionName,stageDescription);
+ os.clearChildren();
+ for (int j = 0; j < stageSpecification.getChildCount(); j++)
+ {
+ ConfigurationNode cn = stageSpecification.findChild(j);
+ os.addChild(os.getChildCount(),new SpecificationNode(cn));
+ }
+ }
else if (childType.equals(JOBNODE_DOCUMENTSPECIFICATION))
{
// Get the job's document specification, clear out the children, and copy new ones from the child.
DocumentSpecification ds = jobDescription.getSpecification();
ds.clearChildren();
- int j = 0;
- while (j < child.getChildCount())
+ for (int j = 0; j < child.getChildCount(); j++)
{
- ConfigurationNode cn = child.findChild(j++);
+ ConfigurationNode cn = child.findChild(j);
ds.addChild(ds.getChildCount(),new SpecificationNode(cn));
}
}
@@ -3609,10 +3955,9 @@ public class ManifoldCF extends org.apac
// Get the job's output specification, clear out the children, and copy new ones from the child.
OutputSpecification os = jobDescription.getOutputSpecification();
os.clearChildren();
- int j = 0;
- while (j < child.getChildCount())
+ for (int j = 0; j < child.getChildCount(); j++)
{
- ConfigurationNode cn = child.findChild(j++);
+ ConfigurationNode cn = child.findChild(j);
os.addChild(os.getChildCount(),new SpecificationNode(cn));
}
}
@@ -3770,7 +4115,6 @@ public class ManifoldCF extends org.apac
{
// For each field of the job, add an appropriate child node, with value.
ConfigurationNode child;
- int j;
// id
if (job.getID() != null)
@@ -3807,10 +4151,9 @@ public class ManifoldCF extends org.apac
// Document specification
DocumentSpecification ds = job.getSpecification();
child = new ConfigurationNode(JOBNODE_DOCUMENTSPECIFICATION);
- j = 0;
- while (j < ds.getChildCount())
+ for (int j = 0; j < ds.getChildCount(); j++)
{
- ConfigurationNode cn = ds.getChild(j++);
+ ConfigurationNode cn = ds.getChild(j);
child.addChild(child.getChildCount(),cn);
}
jobNode.addChild(jobNode.getChildCount(),child);
@@ -3818,14 +4161,39 @@ public class ManifoldCF extends org.apac
// Output specification
OutputSpecification os = job.getOutputSpecification();
child = new ConfigurationNode(JOBNODE_OUTPUTSPECIFICATION);
- j = 0;
- while (j < os.getChildCount())
+ for (int j = 0; j < os.getChildCount(); j++)
{
- ConfigurationNode cn = os.getChild(j++);
+ ConfigurationNode cn = os.getChild(j);
child.addChild(child.getChildCount(),cn);
}
jobNode.addChild(jobNode.getChildCount(),child);
+ // Pipeline stages
+ for (int j = 0; j < job.countPipelineStages(); j++)
+ {
+ child = new ConfigurationNode(JOBNODE_PIPELINESTAGE);
+ ConfigurationNode stage;
+ stage = new ConfigurationNode(JOBNODE_STAGECONNECTIONNAME);
+ stage.setValue(job.getPipelineStageConnectionName(j));
+ child.addChild(child.getChildCount(),stage);
+ String description = job.getPipelineStageDescription(j);
+ if (description != null)
+ {
+ stage = new ConfigurationNode(JOBNODE_STAGEDESCRIPTION);
+ stage.setValue(description);
+ child.addChild(child.getChildCount(),stage);
+ }
+ OutputSpecification spec = job.getPipelineStageSpecification(j);
+ stage = new ConfigurationNode(JOBNODE_STAGESPECIFICATION);
+ for (int k = 0; k < spec.getChildCount(); k++)
+ {
+ ConfigurationNode cn = spec.getChild(k);
+ stage.addChild(stage.getChildCount(),cn);
+ }
+ child.addChild(child.getChildCount(),stage);
+ jobNode.addChild(jobNode.getChildCount(),child);
+ }
+
// Start mode
child = new ConfigurationNode(JOBNODE_STARTMODE);
child.setValue(startModeMap(job.getStartMethod()));
@@ -3897,10 +4265,9 @@ public class ManifoldCF extends org.apac
}
// Schedule records
- j = 0;
- while (j < job.getScheduleRecordCount())
+ for (int j = 0; j < job.getScheduleRecordCount(); j++)
{
- ScheduleRecord sr = job.getScheduleRecord(j++);
+ ScheduleRecord sr = job.getScheduleRecord(j);
child = new ConfigurationNode(JOBNODE_SCHEDULE);
ConfigurationNode recordChild;
@@ -4406,6 +4773,118 @@ public class ManifoldCF extends org.apac
}
+ // Transformation connection API support
+
+ /** Convert input hierarchy into a TransformationConnection object.
+ */
+ protected static void processTransformationConnection(org.apache.manifoldcf.agents.transformationconnection.TransformationConnection connection, ConfigurationNode connectionNode)
+ throws ManifoldCFException
+ {
+ // Walk through the node's children
+ int i = 0;
+ while (i < connectionNode.getChildCount())
+ {
+ ConfigurationNode child = connectionNode.findChild(i++);
+ String childType = child.getType();
+ if (childType.equals(CONNECTIONNODE_ISNEW))
+ {
+ if (child.getValue() == null)
+ throw new ManifoldCFException("Connection isnew node requires a value");
+ connection.setIsNew(child.getValue().equals("true"));
+ }
+ else if (childType.equals(CONNECTIONNODE_NAME))
+ {
+ if (child.getValue() == null)
+ throw new ManifoldCFException("Connection name node requires a value");
+ connection.setName(child.getValue());
+ }
+ else if (childType.equals(CONNECTIONNODE_CLASSNAME))
+ {
+ if (child.getValue() == null)
+ throw new ManifoldCFException("Connection classname node requires a value");
+ connection.setClassName(child.getValue());
+ }
+ else if (childType.equals(CONNECTIONNODE_MAXCONNECTIONS))
+ {
+ if (child.getValue() == null)
+ throw new ManifoldCFException("Connection maxconnections node requires a value");
+ try
+ {
+ connection.setMaxConnections(Integer.parseInt(child.getValue()));
+ }
+ catch (NumberFormatException e)
+ {
+ throw new ManifoldCFException("Error parsing max connections: "+e.getMessage(),e);
+ }
+ }
+ else if (childType.equals(CONNECTIONNODE_DESCRIPTION))
+ {
+ if (child.getValue() == null)
+ throw new ManifoldCFException("Connection description node requires a value");
+ connection.setDescription(child.getValue());
+ }
+ else if (childType.equals(CONNECTIONNODE_CONFIGURATION))
+ {
+ // Get the connection's configuration, clear out the children, and copy new ones from the child.
+ ConfigParams cp = connection.getConfigParams();
+ cp.clearChildren();
+ int j = 0;
+ while (j < child.getChildCount())
+ {
+ ConfigurationNode cn = child.findChild(j++);
+ cp.addChild(cp.getChildCount(),new ConfigNode(cn));
+ }
+ }
+ else
+ throw new ManifoldCFException("Unrecognized output connection field: '"+childType+"'");
+ }
+ if (connection.getClassName() == null)
+ throw new ManifoldCFException("Missing connection field: '"+CONNECTIONNODE_CLASSNAME+"'");
+
+ }
+
+ /** Format a transformation connection.
+ */
+ protected static void formatTransformationConnection(ConfigurationNode connectionNode, ITransformationConnection connection)
+ {
+ ConfigurationNode child;
+ int j;
+
+ child = new ConfigurationNode(CONNECTIONNODE_ISNEW);
+ child.setValue(connection.getIsNew()?"true":"false");
+ connectionNode.addChild(connectionNode.getChildCount(),child);
+
+ child = new ConfigurationNode(CONNECTIONNODE_NAME);
+ child.setValue(connection.getName());
+ connectionNode.addChild(connectionNode.getChildCount(),child);
+
+ child = new ConfigurationNode(CONNECTIONNODE_CLASSNAME);
+ child.setValue(connection.getClassName());
+ connectionNode.addChild(connectionNode.getChildCount(),child);
+
+ child = new ConfigurationNode(CONNECTIONNODE_MAXCONNECTIONS);
+ child.setValue(Integer.toString(connection.getMaxConnections()));
+ connectionNode.addChild(connectionNode.getChildCount(),child);
+
+ if (connection.getDescription() != null)
+ {
+ child = new ConfigurationNode(CONNECTIONNODE_DESCRIPTION);
+ child.setValue(connection.getDescription());
+ connectionNode.addChild(connectionNode.getChildCount(),child);
+ }
+
+ ConfigParams cp = connection.getConfigParams();
+ child = new ConfigurationNode(CONNECTIONNODE_CONFIGURATION);
+ j = 0;
+ while (j < cp.getChildCount())
+ {
+ ConfigurationNode cn = cp.findChild(j++);
+ child.addChild(child.getChildCount(),cn);
+ }
+ connectionNode.addChild(connectionNode.getChildCount(),child);
+
+ }
+
// Authority connection API support
/** Convert input hierarchy into an AuthorityConnection object.
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1601529&r1=1601528&r2=1601529&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java Mon Jun 9 23:19:08 2014
@@ -145,14 +145,22 @@ public class WorkerThread extends Thread
// Universal data, from the job
String connectionName = job.getConnectionName();
String outputName = job.getOutputConnectionName();
- String newParameterVersion = packParameters(job.getForcedMetadata());
+ int pipelineCount = job.countPipelineStages();
+ String[] transformationNames = new String[pipelineCount];
+ OutputSpecification[] transformationSpecifications = new OutputSpecification[pipelineCount];
+ for (int k = 0; k < pipelineCount; k++)
+ {
+ transformationNames[k] = job.getPipelineStageConnectionName(k);
+ transformationSpecifications[k] = job.getPipelineStageSpecification(k);
+ }
+
DocumentSpecification spec = job.getSpecification();
OutputSpecification outputSpec = job.getOutputSpecification();
int jobType = job.getType();
IRepositoryConnection connection = qds.getConnection();
- OutputActivity ingestLogger = new OutputActivity(connectionName,connMgr,outputName);
+ OutputActivity ingestLogger = new OutputActivity(connectionName,connMgr);
// The flow through this section of the code is as follows.
// (1) We start with a list of documents
@@ -307,14 +315,23 @@ public class WorkerThread extends Thread
}
}
- // Get the output version string.
- String outputVersion = ingester.getOutputDescription(outputName,outputSpec);
-
- HashMap abortSet = new HashMap();
- VersionActivity versionActivity = new VersionActivity(processID,connectionName,connMgr,jobManager,job,ingester,abortSet,outputVersion);
+ // Get the output version string. Cannot be null.
+ String outputDescriptionString = ingester.getOutputDescription(outputName,outputSpec);
+ // Get the transformation version strings. Cannot be null.
+ String[] transformationDescriptionStrings = ingester.getTransformationDescriptions(transformationNames,transformationSpecifications);
+
+ // New version strings
+ String newOutputVersion = outputDescriptionString;
+ String newParameterVersion = packParameters(job.getForcedMetadata());
+ String newTransformationVersion = packTransformations(transformationNames,transformationDescriptionStrings);
+
+ Set<String> abortSet = new HashSet<String>();
+ VersionActivity versionActivity = new VersionActivity(job.getID(),processID,connectionName,outputName,transformationNames,connMgr,jobManager,ingester,abortSet,outputDescriptionString,transformationDescriptionStrings,ingestLogger);
String aclAuthority = connection.getACLAuthority();
- boolean isDefaultAuthority = (aclAuthority == null || aclAuthority.length() == 0);
+ if (aclAuthority == null)
+ aclAuthority = "";
+ boolean isDefaultAuthority = (aclAuthority.length() == 0);
if (Logging.threads.isDebugEnabled())
Logging.threads.debug("Worker thread getting versions for "+Integer.toString(currentDocIDArray.length)+" documents");
@@ -390,10 +407,6 @@ public class WorkerThread extends Thread
// This try{ } is for releasing document versions at the connector level.
try
{
- // Organize what we need for document status comparison, and get it into a canonical form.
- String newOutputVersion = outputVersion;
- if (newOutputVersion == null)
- newOutputVersion = "";
// Loop through documents now, and amass what we need to fetch.
// We also need to tally: (1) what needs to be marked as deleted via
@@ -405,7 +418,7 @@ public class WorkerThread extends Thread
QueuedDocument qd = activeDocuments.get(i);
DocumentDescription dd = qd.getDocumentDescription();
// If this document was aborted, then treat it specially; we never go on to fetch it, for one thing.
- if (abortSet.get(dd.getDocumentIdentifier()) != null)
+ if (abortSet.contains(dd.getDocumentIdentifier()))
{
// Special treatment for aborted documents.
// We ignore the returned version string completely, since it's presumed that processing was not completed for this doc.
@@ -419,9 +432,6 @@ public class WorkerThread extends Thread
DocumentIngestStatus oldDocStatus = qd.getLastIngestedStatus();
String documentIDHash = dd.getDocumentIdentifierHash();
String newDocVersion = newVersionStringArray[i];
- String newAuthorityName = aclAuthority;
- if (newAuthorityName == null)
- newAuthorityName = "";
versionMap.put(dd.getDocumentIdentifierHash(),newDocVersion);
@@ -448,17 +458,10 @@ public class WorkerThread extends Thread
// that was there before is there now (which may mean a rescan),
// or (2) there are different versions (which ALWAYS means a rescan).
String oldDocVersion = oldDocStatus.getDocumentVersion();
- if (oldDocVersion == null)
- oldDocVersion = "";
String oldAuthorityName = oldDocStatus.getDocumentAuthorityNameString();
- if (oldAuthorityName == null)
- oldAuthorityName = "";
String oldOutputVersion = oldDocStatus.getOutputVersion();
- if (oldOutputVersion == null)
- oldOutputVersion = "";
+ String oldTransformationVersion = oldDocStatus.getTransformationVersion();
String oldParameterVersion = oldDocStatus.getParameterVersion();
- if (oldParameterVersion == null)
- oldParameterVersion = "";
// Start the comparison processing
if (newDocVersion.length() == 0)
@@ -467,8 +470,9 @@ public class WorkerThread extends Thread
allowIngest = true;
}
else if (oldDocVersion.equals(newDocVersion) &&
- oldAuthorityName.equals(newAuthorityName) &&
+ oldAuthorityName.equals(aclAuthority) &&
oldOutputVersion.equals(newOutputVersion) &&
+ oldTransformationVersion.equals(newTransformationVersion) &&
oldParameterVersion.equals(newParameterVersion))
{
// The old logic was as follows:
@@ -527,9 +531,18 @@ public class WorkerThread extends Thread
}
// First, make the things we will need for all subsequent steps.
- ProcessActivity activity = new ProcessActivity(processID,
+ ProcessActivity activity = new ProcessActivity(job.getID(),processID,
threadContext,rt,jobManager,ingester,
- currentTime,job,connection,connector,connMgr,legalLinkTypes,ingestLogger,abortSet,outputVersion,newParameterVersion);
+ connectionName,outputName,transformationNames,
+ outputDescriptionString,transformationDescriptionStrings,
+ currentTime,
+ job.getExpiration(),
+ job.getForcedMetadata(),
+ job.getInterval(),
+ job.getMaxInterval(),
+ job.getHopcountMode(),
+ connection,connector,connMgr,legalLinkTypes,ingestLogger,abortSet,
+ newOutputVersion,newTransformationVersion,newParameterVersion);
try
{
@@ -677,7 +690,7 @@ public class WorkerThread extends Thread
String documentID = recrawlDocs[i].getDocumentIdentifier();
// If aborted due to sequencing issue, then requeue for reprocessing immediately, ignoring everything else.
- boolean wasAborted = abortSet.get(documentID) != null;
+ boolean wasAborted = abortSet.contains(documentID);
if (wasAborted)
{
// Requeue for immediate reprocessing
@@ -742,7 +755,7 @@ public class WorkerThread extends Thread
{
QueuedDocument qd = finishList.get(i);
DocumentDescription dd = qd.getDocumentDescription();
- if (abortSet.get(dd.getDocumentIdentifier()) != null)
+ if (abortSet.contains(dd.getDocumentIdentifier()))
{
// The document was aborted, so put it into the abortedList
abortedList.add(dd);
@@ -1186,6 +1199,25 @@ public class WorkerThread extends Thread
}
}
+ protected static String packTransformations(String[] transformationNames, String[] transformationDescriptionStrings)
+ {
+ StringBuilder sb = new StringBuilder();
+ packList(sb,transformationNames,'+');
+ packList(sb,transformationDescriptionStrings,'!');
+ return sb.toString();
+ }
+
+ /** Another stuffer for packing lists of variable length */
+ protected static void packList(StringBuilder output, String[] values, char delimiter)
+ {
+ pack(output,Integer.toString(values.length),delimiter);
+ int i = 0;
+ while (i < values.length)
+ {
+ pack(output,values[i++],delimiter);
+ }
+ }
+
protected static String packParameters(Map<String,Set<String>> forcedParameters)
{
StringBuilder sb = new StringBuilder();
@@ -1238,59 +1270,82 @@ public class WorkerThread extends Thread
*/
protected static class VersionActivity implements IVersionActivity
{
+ protected final Long jobID;
protected final String processID;
protected final String connectionName;
+ protected final String outputConnectionName;
+ protected final String[] transformationConnectionNames;
protected final IRepositoryConnectionManager connMgr;
protected final IJobManager jobManager;
- protected final IJobDescription job;
protected final IIncrementalIngester ingester;
- protected final HashMap abortSet;
- protected final String outputVersion;
-
+ protected final Set<String> abortSet;
+ protected final String outputDescriptionString;
+ protected final String[] transformationDescriptionStrings;
+ protected final CheckActivity checkActivity;
/** Constructor.
*/
- public VersionActivity(String processID, String connectionName, IRepositoryConnectionManager connMgr,
- IJobManager jobManager, IJobDescription job, IIncrementalIngester ingester, HashMap abortSet,
- String outputVersion)
+ public VersionActivity(Long jobID, String processID,
+ String connectionName, String outputConnectionName,
+ String[] transformationConnectionNames,
+ IRepositoryConnectionManager connMgr,
+ IJobManager jobManager, IIncrementalIngester ingester, Set<String> abortSet,
+ String outputDescriptionString, String[] transformationDescriptionStrings,
+ CheckActivity checkActivity)
{
+ this.jobID = jobID;
this.processID = processID;
this.connectionName = connectionName;
+ this.outputConnectionName = outputConnectionName;
+ this.transformationConnectionNames = transformationConnectionNames;
this.connMgr = connMgr;
this.jobManager = jobManager;
- this.job = job;
this.ingester = ingester;
this.abortSet = abortSet;
- this.outputVersion = outputVersion;
+ this.outputDescriptionString = outputDescriptionString;
+ this.transformationDescriptionStrings = transformationDescriptionStrings;
+ this.checkActivity = checkActivity;
}
/** Check whether a mime type is indexable by the currently specified output connector.
*@param mimeType is the mime type to check, not including any character set specification.
*@return true if the mime type is indexable.
*/
+ @Override
public boolean checkMimeTypeIndexable(String mimeType)
throws ManifoldCFException, ServiceInterruption
{
- return ingester.checkMimeTypeIndexable(job.getOutputConnectionName(),outputVersion,mimeType);
+ return ingester.checkMimeTypeIndexable(
+ transformationConnectionNames,transformationDescriptionStrings,
+ outputConnectionName,outputDescriptionString,mimeType,
+ checkActivity);
}
/** Check whether a document is indexable by the currently specified output connector.
*@param localFile is the local copy of the file to check.
*@return true if the document is indexable.
*/
+ @Override
public boolean checkDocumentIndexable(File localFile)
throws ManifoldCFException, ServiceInterruption
{
- return ingester.checkDocumentIndexable(job.getOutputConnectionName(),outputVersion,localFile);
+ return ingester.checkDocumentIndexable(
+ transformationConnectionNames,transformationDescriptionStrings,
+ outputConnectionName,outputDescriptionString,localFile,
+ checkActivity);
}
/** Check whether a document of a specified length is indexable by the currently specified output connector.
*@param length is the length to check.
*@return true if the document is indexable.
*/
+ @Override
public boolean checkLengthIndexable(long length)
throws ManifoldCFException, ServiceInterruption
{
- return ingester.checkLengthIndexable(job.getOutputConnectionName(),outputVersion,length);
+ return ingester.checkLengthIndexable(
+ transformationConnectionNames,transformationDescriptionStrings,
+ outputConnectionName,outputDescriptionString,length,
+ checkActivity);
}
/** Pre-determine whether a document's URL is indexable by this connector. This method is used by participating repository connectors
@@ -1298,10 +1353,14 @@ public class WorkerThread extends Thread
*@param url is the URL of the document.
*@return true if the file is indexable.
*/
+ @Override
public boolean checkURLIndexable(String url)
throws ManifoldCFException, ServiceInterruption
{
- return ingester.checkURLIndexable(job.getOutputConnectionName(),outputVersion,url);
+ return ingester.checkURLIndexable(
+ transformationConnectionNames,transformationDescriptionStrings,
+ outputConnectionName,outputDescriptionString,url,
+ checkActivity);
}
/** Record time-stamped information about the activity of the connector.
@@ -1320,6 +1379,7 @@ public class WorkerThread extends Thread
* described in the resultCode field. This field is not meant to be queried on. May be null.
*@param childIdentifiers is a set of child entity identifiers associated with this activity. May be null.
*/
+ @Override
public void recordActivity(Long startTime, String activityType, Long dataSize,
String entityIdentifier, String resultCode, String resultDescription, String[] childIdentifiers)
throws ManifoldCFException
@@ -1333,10 +1393,11 @@ public class WorkerThread extends Thread
*@param dataName is the name of the data items to retrieve.
*@return an array containing the unique data values passed from ALL parents. Note that these are in no particular order, and there will not be any duplicates.
*/
+ @Override
public String[] retrieveParentData(String localIdentifier, String dataName)
throws ManifoldCFException
{
- return jobManager.retrieveParentData(job.getID(),ManifoldCF.hash(localIdentifier),dataName);
+ return jobManager.retrieveParentData(jobID,ManifoldCF.hash(localIdentifier),dataName);
}
/** Retrieve data passed from parents to a specified child document.
@@ -1344,10 +1405,11 @@ public class WorkerThread extends Thread
*@param dataName is the name of the data items to retrieve.
*@return an array containing the unique data values passed from ALL parents. Note that these are in no particular order, and there will not be any duplicates.
*/
+ @Override
public CharacterInput[] retrieveParentDataAsFiles(String localIdentifier, String dataName)
throws ManifoldCFException
{
- return jobManager.retrieveParentDataAsFiles(job.getID(),ManifoldCF.hash(localIdentifier),dataName);
+ return jobManager.retrieveParentDataAsFiles(jobID,ManifoldCF.hash(localIdentifier),dataName);
}
/** Check whether current job is still active.
@@ -1355,10 +1417,11 @@ public class WorkerThread extends Thread
* itself being aborted. If the connector should abort, this method will raise a properly-formed ServiceInterruption, which if thrown to the
* caller, will signal that the current versioning activity remains incomplete and must be retried when the job is resumed.
*/
+ @Override
public void checkJobStillActive()
throws ManifoldCFException, ServiceInterruption
{
- if (jobManager.checkJobActive(job.getID()) == false)
+ if (jobManager.checkJobActive(jobID) == false)
throw new ServiceInterruption("Job no longer active",System.currentTimeMillis(),true);
}
@@ -1369,6 +1432,7 @@ public class WorkerThread extends Thread
*@param eventName is the event name.
*@return false if the event is already in the "pending" state.
*/
+ @Override
public boolean beginEventSequence(String eventName)
throws ManifoldCFException
{
@@ -1382,6 +1446,7 @@ public class WorkerThread extends Thread
* the sole right to complete it. Otherwise, race conditions can develop which would be difficult to diagnose.
*@param eventName is the event name.
*/
+ @Override
public void completeEventSequence(String eventName)
throws ManifoldCFException
{
@@ -1394,17 +1459,19 @@ public class WorkerThread extends Thread
* presumed that the reason for the requeue is because of sequencing issues synchronized around an underlying event.
*@param localIdentifier is the document identifier to requeue
*/
+ @Override
public void retryDocumentProcessing(String localIdentifier)
throws ManifoldCFException
{
// Accumulate aborts
- abortSet.put(localIdentifier,localIdentifier);
+ abortSet.add(localIdentifier);
}
/** Create a global string from a simple string.
*@param simpleString is the simple string.
*@return a global string.
*/
+ @Override
public String createGlobalString(String simpleString)
{
return ManifoldCF.createGlobalString(simpleString);
@@ -1414,6 +1481,7 @@ public class WorkerThread extends Thread
*@param simpleString is the simple string.
*@return a connection-specific string.
*/
+ @Override
public String createConnectionSpecificString(String simpleString)
{
return ManifoldCF.createConnectionSpecificString(connectionName,simpleString);
@@ -1423,9 +1491,10 @@ public class WorkerThread extends Thread
*@param simpleString is the simple string.
*@return a job-specific string.
*/
+ @Override
public String createJobSpecificString(String simpleString)
{
- return ManifoldCF.createJobSpecificString(job.getID(),simpleString);
+ return ManifoldCF.createJobSpecificString(jobID,simpleString);
}
}
@@ -1435,20 +1504,31 @@ public class WorkerThread extends Thread
protected static class ProcessActivity implements IProcessActivity
{
// Member variables
+ protected final Long jobID;
protected final String processID;
protected final IThreadContext threadContext;
protected final IJobManager jobManager;
protected final IIncrementalIngester ingester;
+ protected final String connectionName;
+ protected final String outputName;
+ protected final String[] transformationConnectionNames;
+ protected final String outputDescriptionString;
+ protected final String[] transformationDescriptionStrings;
protected final long currentTime;
- protected final IJobDescription job;
+ protected final Long expireInterval;
+ protected final Map<String,Set<String>> forcedMetadata;
+ protected final Long recrawlInterval;
+ protected final Long maxInterval;
+ protected final int hopcountMode;
protected final IRepositoryConnection connection;
protected final IRepositoryConnector connector;
protected final IRepositoryConnectionManager connMgr;
protected final String[] legalLinkTypes;
protected final OutputActivity ingestLogger;
protected final IReprioritizationTracker rt;
- protected final HashMap abortSet;
+ protected final Set<String> abortSet;
protected final String outputVersion;
+ protected final String transformationVersion;
protected final String parameterVersion;
// We submit references in bulk, because that's way more efficient.
@@ -1467,20 +1547,40 @@ public class WorkerThread extends Thread
*@param jobManager is the job manager
*@param ingester is the ingester
*/
- public ProcessActivity(String processID, IThreadContext threadContext,
+ public ProcessActivity(Long jobID, String processID,
+ IThreadContext threadContext,
IReprioritizationTracker rt, IJobManager jobManager,
- IIncrementalIngester ingester, long currentTime,
- IJobDescription job, IRepositoryConnection connection, IRepositoryConnector connector,
+ IIncrementalIngester ingester,
+ String connectionName, String outputName, String[] transformationConnectionNames,
+ String outputDescriptionString, String[] transformationDescriptionStrings,
+ long currentTime,
+ Long expireInterval,
+ Map<String,Set<String>> forcedMetadata,
+ Long recrawlInterval,
+ Long maxInterval,
+ int hopcountMode,
+ IRepositoryConnection connection, IRepositoryConnector connector,
IRepositoryConnectionManager connMgr, String[] legalLinkTypes, OutputActivity ingestLogger,
- HashMap abortSet, String outputVersion, String parameterVersion)
+ Set<String> abortSet,
+ String outputVersion, String transformationVersion, String parameterVersion)
{
+ this.jobID = jobID;
this.processID = processID;
this.threadContext = threadContext;
this.rt = rt;
this.jobManager = jobManager;
this.ingester = ingester;
+ this.connectionName = connectionName;
+ this.outputName = outputName;
+ this.outputDescriptionString = outputDescriptionString;
+ this.transformationConnectionNames = transformationConnectionNames;
+ this.transformationDescriptionStrings = transformationDescriptionStrings;
this.currentTime = currentTime;
- this.job = job;
+ this.expireInterval = expireInterval;
+ this.forcedMetadata = forcedMetadata;
+ this.recrawlInterval = recrawlInterval;
+ this.maxInterval = maxInterval;
+ this.hopcountMode = hopcountMode;
this.connection = connection;
this.connector = connector;
this.connMgr = connMgr;
@@ -1489,6 +1589,7 @@ public class WorkerThread extends Thread
this.abortSet = abortSet;
this.outputVersion = outputVersion;
this.parameterVersion = parameterVersion;
+ this.transformationVersion = transformationVersion;
}
/** Clean up any dangling information, before abandoning this process activity object */
@@ -1531,7 +1632,7 @@ public class WorkerThread extends Thread
+" to '"+localIdentifier+"', relationship type "+((relationshipType==null)?"null":"'"+relationshipType+"'")
+", with "+((dataNames==null)?"no":Integer.toString(dataNames.length))+" data types, origination time="+((originationTime==null)?"unknown":originationTime.toString()));
- Long expireInterval = job.getExpiration();
+ //Long expireInterval = job.getExpiration();
if (expireInterval != null)
{
// We should not queue documents that have already expired; it wastes time
@@ -1679,7 +1780,7 @@ public class WorkerThread extends Thread
public String[] retrieveParentData(String localIdentifier, String dataName)
throws ManifoldCFException
{
- return jobManager.retrieveParentData(job.getID(),ManifoldCF.hash(localIdentifier),dataName);
+ return jobManager.retrieveParentData(jobID,ManifoldCF.hash(localIdentifier),dataName);
}
/** Retrieve data passed from parents to a specified child document.
@@ -1691,7 +1792,7 @@ public class WorkerThread extends Thread
public CharacterInput[] retrieveParentDataAsFiles(String localIdentifier, String dataName)
throws ManifoldCFException
{
- return jobManager.retrieveParentDataAsFiles(job.getID(),ManifoldCF.hash(localIdentifier),dataName);
+ return jobManager.retrieveParentDataAsFiles(jobID,ManifoldCF.hash(localIdentifier),dataName);
}
/** Record a document version, but don't ingest it.
@@ -1704,7 +1805,7 @@ public class WorkerThread extends Thread
throws ManifoldCFException, ServiceInterruption
{
String documentIdentifierHash = ManifoldCF.hash(documentIdentifier);
- ingester.documentRecord(job.getOutputConnectionName(),job.getConnectionName(),documentIdentifierHash,version,currentTime,ingestLogger);
+ ingester.documentRecord(outputName,connectionName,documentIdentifierHash,version,currentTime,ingestLogger);
}
/** Ingest the current document.
@@ -1727,7 +1828,7 @@ public class WorkerThread extends Thread
if (data != null)
{
- Map<String,Set<String>> forcedMetadata = job.getForcedMetadata();
+ //Map<String,Set<String>> forcedMetadata = job.getForcedMetadata();
// Modify the repository document with forced parameters.
for (String paramName : forcedMetadata.keySet())
@@ -1744,9 +1845,12 @@ public class WorkerThread extends Thread
}
// First, we need to add into the metadata the stuff from the job description.
- ingester.documentIngest(job.getOutputConnectionName(),
- job.getConnectionName(),documentIdentifierHash,
- version,outputVersion,parameterVersion,
+ ingester.documentIngest(transformationConnectionNames,
+ transformationDescriptionStrings,
+ outputName,
+ outputDescriptionString,
+ connectionName,documentIdentifierHash,
+ version,transformationVersion,outputVersion,parameterVersion,
connection.getACLAuthority(),
data,currentTime,
documentURI,
@@ -1780,8 +1884,8 @@ public class WorkerThread extends Thread
throws ManifoldCFException, ServiceInterruption
{
String documentIdentifierHash = ManifoldCF.hash(documentIdentifier);
- ingester.documentDelete(job.getOutputConnectionName(),
- job.getConnectionName(),documentIdentifierHash,
+ ingester.documentDelete(outputName,
+ connectionName,documentIdentifierHash,
ingestLogger);
}
@@ -1868,10 +1972,10 @@ public class WorkerThread extends Thread
public Long calculateDocumentRescheduleTime(long currentTime, long timeAmt, String localIdentifier)
{
Long recrawlTime = null;
- Long recrawlInterval = job.getInterval();
+ //Long recrawlInterval = job.getInterval();
if (recrawlInterval != null)
{
- Long maxInterval = job.getMaxInterval();
+ //Long maxInterval = job.getMaxInterval();
long actualInterval = recrawlInterval.longValue() + timeAmt;
if (maxInterval != null && actualInterval > maxInterval.longValue())
actualInterval = maxInterval.longValue();
@@ -1908,7 +2012,7 @@ public class WorkerThread extends Thread
Long originationTime = getDocumentOriginationTime(localIdentifier);
if (originationTime == null)
originationTime = new Long(currentTime);
- Long expireInterval = job.getExpiration();
+ //Long expireInterval = job.getExpiration();
Long expireTime = null;
if (expireInterval != null)
expireTime = new Long(originationTime.longValue() + expireInterval.longValue());
@@ -2024,7 +2128,7 @@ public class WorkerThread extends Thread
rt.preloadBinValues();
jobManager.addDocuments(processID,
- job.getID(),legalLinkTypes,docidHashes,docids,db.getParentIdentifierHash(),db.getLinkType(),job.getHopcountMode(),
+ jobID,legalLinkTypes,docidHashes,docids,db.getParentIdentifierHash(),db.getLinkType(),hopcountMode,
dataNames,dataValues,currentTime,priorities,eventNames);
rt.clearPreloadedValues();
@@ -2042,7 +2146,7 @@ public class WorkerThread extends Thread
public void checkJobStillActive()
throws ManifoldCFException, ServiceInterruption
{
- if (jobManager.checkJobActive(job.getID()) == false)
+ if (jobManager.checkJobActive(jobID) == false)
throw new ServiceInterruption("Job no longer active",System.currentTimeMillis());
}
@@ -2085,7 +2189,7 @@ public class WorkerThread extends Thread
throws ManifoldCFException
{
// Accumulate aborts
- abortSet.put(localIdentifier,localIdentifier);
+ abortSet.add(localIdentifier);
}
/** Check whether a mime type is indexable by the currently specified output connector.
@@ -2096,7 +2200,10 @@ public class WorkerThread extends Thread
public boolean checkMimeTypeIndexable(String mimeType)
throws ManifoldCFException, ServiceInterruption
{
- return ingester.checkMimeTypeIndexable(job.getOutputConnectionName(),outputVersion,mimeType);
+ return ingester.checkMimeTypeIndexable(
+ transformationConnectionNames,transformationDescriptionStrings,
+ outputName,outputDescriptionString,mimeType,
+ ingestLogger);
}
/** Check whether a document is indexable by the currently specified output connector.
@@ -2107,7 +2214,10 @@ public class WorkerThread extends Thread
public boolean checkDocumentIndexable(File localFile)
throws ManifoldCFException, ServiceInterruption
{
- return ingester.checkDocumentIndexable(job.getOutputConnectionName(),outputVersion,localFile);
+ return ingester.checkDocumentIndexable(
+ transformationConnectionNames,transformationDescriptionStrings,
+ outputName,outputDescriptionString,localFile,
+ ingestLogger);
}
/** Check whether a document of a specified length is indexable by the currently specified output connector.
@@ -2118,7 +2228,10 @@ public class WorkerThread extends Thread
public boolean checkLengthIndexable(long length)
throws ManifoldCFException, ServiceInterruption
{
- return ingester.checkLengthIndexable(job.getOutputConnectionName(),outputVersion,length);
+ return ingester.checkLengthIndexable(
+ transformationConnectionNames,transformationDescriptionStrings,
+ outputName,outputDescriptionString,length,
+ ingestLogger);
}
/** Pre-determine whether a document's URL is indexable by this connector. This method is used by participating repository connectors
@@ -2130,7 +2243,10 @@ public class WorkerThread extends Thread
public boolean checkURLIndexable(String url)
throws ManifoldCFException, ServiceInterruption
{
- return ingester.checkURLIndexable(job.getOutputConnectionName(),outputVersion,url);
+ return ingester.checkURLIndexable(
+ transformationConnectionNames,transformationDescriptionStrings,
+ outputName,outputDescriptionString,url,
+ ingestLogger);
}
/** Create a global string from a simple string.
@@ -2160,7 +2276,7 @@ public class WorkerThread extends Thread
@Override
public String createJobSpecificString(String simpleString)
{
- return ManifoldCF.createJobSpecificString(job.getID(),simpleString);
+ return ManifoldCF.createJobSpecificString(jobID,simpleString);
}
}
@@ -2428,23 +2544,78 @@ public class WorkerThread extends Thread
}
}
+ /** The check activity class */
+ protected static class CheckActivity implements IOutputCheckActivity
+ {
+ public CheckActivity()
+ {
+ }
+
+ /** Detect if a mime type is acceptable downstream or not. This method is used to determine whether it makes sense to fetch a document
+ * in the first place.
+ *@param mimeType is the mime type of the document.
+ *@return true if the mime type can be accepted by the downstream connection.
+ */
+ @Override
+ public boolean checkMimeTypeIndexable(String mimeType)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ return false;
+ }
+
+ /** Pre-determine whether a document (passed here as a File object) is acceptable downstream. This method is
+ * used to determine whether a document needs to be actually transferred. This hook is provided mainly to support
+ * search engines that only handle a small set of accepted file types.
+ *@param localFile is the local file to check.
+ *@return true if the file is acceptable by the downstream connection.
+ */
+ @Override
+ public boolean checkDocumentIndexable(File localFile)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ return false;
+ }
+
+ /** Pre-determine whether a document's length is acceptable downstream. This method is used
+ * to determine whether to fetch a document in the first place.
+ *@param length is the length of the document.
+ *@return true if the file is acceptable by the downstream connection.
+ */
+ @Override
+ public boolean checkLengthIndexable(long length)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ return false;
+ }
+
+ /** Pre-determine whether a document's URL is acceptable downstream. This method is used
+ * to help filter out documents that cannot be indexed in advance.
+ *@param url is the URL of the document.
+ *@return true if the file is acceptable by the downstream connection.
+ */
+ @Override
+ public boolean checkURLIndexable(String url)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ return false;
+ }
+
+ }
+
/** The ingest logger class */
- protected static class OutputActivity implements IOutputActivity
+ protected static class OutputActivity extends CheckActivity implements IOutputActivity
{
// Connection name
- protected String connectionName;
+ protected final String connectionName;
// Connection manager
- protected IRepositoryConnectionManager connMgr;
- // Output connection name
- protected String outputConnectionName;
+ protected final IRepositoryConnectionManager connMgr;
/** Constructor */
- public OutputActivity(String connectionName, IRepositoryConnectionManager connMgr, String outputConnectionName)
+ public OutputActivity(String connectionName, IRepositoryConnectionManager connMgr)
{
this.connectionName = connectionName;
this.connMgr = connMgr;
- this.outputConnectionName = outputConnectionName;
}
/** Record time-stamped information about the activity of the output connector.
@@ -2462,11 +2633,12 @@ public class WorkerThread extends Thread
*@param resultDescription is a (possibly long) human-readable string which adds detail, if required, to the result
* described in the resultCode field. This field is not meant to be queried on. May be null.
*/
+ @Override
public void recordActivity(Long startTime, String activityType, Long dataSize,
String entityURI, String resultCode, String resultDescription)
throws ManifoldCFException
{
- connMgr.recordHistory(connectionName,startTime,ManifoldCF.qualifyOutputActivityName(activityType,outputConnectionName),dataSize,entityURI,resultCode,
+ connMgr.recordHistory(connectionName,startTime,activityType,dataSize,entityURI,resultCode,
resultDescription,null);
}
@@ -2476,6 +2648,7 @@ public class WorkerThread extends Thread
*@param accessToken is the raw, repository access token.
*@return the properly qualified access token.
*/
+ @Override
public String qualifyAccessToken(String authorityNameString, String accessToken)
throws ManifoldCFException
{
@@ -2485,6 +2658,19 @@ public class WorkerThread extends Thread
return URLEncoder.encode(authorityNameString) + ":" + URLEncoder.encode(accessToken);
}
+ /** Send a document via the pipeline to the next output connection.
+ *@param documentURI is the document's URI.
+ *@param document is the document data to be processed (handed to the output data store).
+ *@param authorityNameString is the authority name string that should be used to qualify the document's access tokens.
+ *@return the document status (accepted or permanently rejected); return codes are listed in IPipelineConnector.
+ */
+ public int sendDocument(String documentURI, RepositoryDocument document, String authorityNameString)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ // No downstream connection at output connection level.
+ return IPipelineConnector.DOCUMENTSTATUS_REJECTED;
+ }
+
}
}
Modified: manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties?rev=1601529&r1=1601528&r2=1601529&view=diff
==============================================================================
--- manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties (original)
+++ manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties Mon Jun 9 23:19:08 2014
@@ -13,9 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-editoutput.Name=Name
-editoutput.Type=Type
-editoutput.Throttling=Throttling
editconnection.Name=Name
editconnection.Type=Type
editconnection.Throttling=Throttling
@@ -48,6 +45,7 @@ banner.DocumentIngestion=Document Ingest
navigation.LogOut=Log Out
navigation.Outputs=Outputs
+navigation.ListTransformationConnections=List Transformation Connections
navigation.ListOutputConnections=List Output Connections
navigation.Authorities=Authorities
navigation.ListUserMappings=List User Mapping Connections
@@ -69,6 +67,7 @@ navigation.ResultHistogram=Result Histog
navigation.Miscellaneous=Miscellaneous
navigation.Locale=en_US
navigation.Help=Help
+navigation.Listtransformationconnections=List transformation connections
navigation.Listoutputconnections=List output connections
navigation.Listrepositoryconnections=List repository connections
navigation.Listusermappings=List user mapping connections
@@ -98,6 +97,9 @@ listoutputs.AddAnOutputConnection=Add an
listoutputs.uninstalled=(uninstalled)
editoutput.ApacheManifoldCFEditOutputConnection=Apache ManifoldCF: Edit Output Connection
+editoutput.Name=Name
+editoutput.Type=Type
+editoutput.Throttling=Throttling
editoutput.EditAnOutputConnection=Edit an Output Connection
editoutput.EditOutputConnection2=Edit Output Connection
editoutput.ConnectionTypeColon=Connection type:
@@ -113,7 +115,6 @@ editoutput.NameColon=Name:
editoutput.DescriptionColon=Description:
editoutput.ConnectionMustHaveAName=Connection must have a name
editoutput.TheMaximumNumberOfConnectionsMustBeAValidInteger=The maximum number of connections must be a valid integer
-
editoutput.UNREGISTERED=UNREGISTERED
editoutput.tab=tab
@@ -143,6 +144,60 @@ viewoutput.uninstalled=(uninstalled)
viewoutput.Connectorisnotinstalled=Connector is not installed.
viewoutput.Threwexception=Threw exception:
+listtransformations.ApacheManifoldCFListTransformationConnections=Apache ManifoldCF: List Transformation Connections
+listtransformations.ListOfTransformationConnections=List of Transformation Connections
+listtransformations.Name=Name
+listtransformations.Description=Description
+listtransformations.ConnectionType=Connection Type
+listtransformations.Max=Max
+listtransformations.AddaNewTransformationConnection=Add a new transformation connection
+listtransformations.View=View
+listtransformations.Edit=Edit
+listtransformations.Delete=Delete
+listtransformations.DeleteTransformationConnection=Delete transformation connection
+listtransformations.AddATransformationConnection=Add a transformation connection
+listtransformations.uninstalled=(uninstalled)
+
+edittransformation.ApacheManifoldCFEditTransformationConnection=Apache ManifoldCF: Edit Transformation Connection
+edittransformation.Name=Name
+edittransformation.Type=Type
+edittransformation.Throttling=Throttling
+edittransformation.EditATransformationConnection=Edit a Transformation Connection
+edittransformation.EditTransformationConnection2=Edit Transformation Connection
+edittransformation.ConnectionTypeColon=Connection type:
+edittransformation.Continue=Continue
+edittransformation.ContinueToNextPage=Continue to next page
+edittransformation.Cancel=Cancel
+edittransformation.CancelTransformationConnectionEditing=Cancel transformation connection editing
+edittransformation.Save=Save
+edittransformation.SaveThisTransformationConnection=Save this transformation connection
+edittransformation.MaxConnectionsColon=Max connections:
+edittransformation.EditTransformationConnection=Edit transformation connection
+edittransformation.NameColon=Name:
+edittransformation.DescriptionColon=Description:
+edittransformation.ConnectionMustHaveAName=Connection must have a name
+edittransformation.TheMaximumNumberOfConnectionsMustBeAValidInteger=The maximum number of connections must be a valid integer
+edittransformation.UNREGISTERED=UNREGISTERED
+edittransformation.tab=tab
+
+viewtransformation.ApacheManifoldCFViewTransformationConnectionStatus=Apache ManifoldCF: View Transformation Connection Status
+viewtransformation.ViewTransformationConnectionStatus=View Transformation Connection Status
+viewtransformation.NameColon=Name:
+viewtransformation.DescriptionColon=Description:
+viewtransformation.ConnectionTypeColon=Connection type:
+viewtransformation.MaxConnectionsColon=Max connections:
+viewtransformation.ConnectionStatusColon=Connection status:
+viewtransformation.Refresh=Refresh
+viewtransformation.EditThisTransformationConnection=Edit this transformation connection
+viewtransformation.Edit=Edit
+viewtransformation.DeleteThisTransformationConnection=Delete this transformation connection
+viewtransformation.Delete=Delete
+viewtransformation.Deletetransformationconnection=Delete transformation connection
+viewtransformation.qmark=?
+viewtransformation.uninstalled=(uninstalled)
+viewtransformation.Connectorisnotinstalled=Connector is not installed.
+viewtransformation.Threwexception=Threw exception:
+
listgroups.ApacheManifoldCFListAuthorityGroups=Apache ManifoldCF: List Authority Groups
listgroups.DeleteAuthorityGroup=Delete authority group
listgroups.ListOfAuthorityGroups=List of Authority Groups
@@ -345,6 +400,18 @@ listjobs.CopyJob=Copy job
editjob.ApacheManifoldCFEditJob=Apache ManifoldCF: Edit Job
editjob.EditaJob=Edit a Job
editjob.EditJob=Edit job
+
+editjob.PipelineColon=Pipeline:
+editjob.StageNumber=Stage
+editjob.StageDescription=Description
+editjob.StageConnectionName=Connection name
+editjob.RepositoryStage=(Repository)
+editjob.OutputStage=(Output)
+editjob.InsertBefore=Insert before
+editjob.Insertnewstagehere=Insert new stage here
+editjob.SelectAPipelineStageConnectionName=Select a pipeline stage connection name
+editjob.Deletestage=Delete stage
+
editjob.NoRepositoryConnectionsDefinedCreateOneFirst=No repository connections defined - create one first
editjob.NoOutputConnectionsDefinedCreateOneFirst=No output connections defined - create one first
editjob.NameColon=Name:
@@ -774,6 +841,14 @@ error.Return=Return
viewjob.ApacheManifoldCFViewJob=Apache ManifoldCF: View Job
viewjob.ViewAJob=View a Job
+
+viewjob.PipelineColon=Pipeline:
+viewjob.StageNumber=Stage
+viewjob.StageDescription=Description
+viewjob.StageConnectionName=Connection name
+viewjob.RepositoryStage=(Repository)
+viewjob.OutputStage=(Output)
+
viewjob.ScheduleTypeColon=Schedule type:
viewjob.MinimumRecrawlIntervalColon=Minimum recrawl interval:
viewjob.MaximumRecrawlIntervalColon=Maximum recrawl interval: