You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/06/10 01:19:10 UTC

svn commit: r1601529 [5/6] - in /manifoldcf/trunk: ./ connectors/ connectors/nulltransformation/ framework/agents/src/main/java/org/apache/manifoldcf/agents/ framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/ framework/agent...

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java?rev=1601529&r1=1601528&r2=1601529&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java Mon Jun  9 23:19:08 2014
@@ -50,7 +50,8 @@ public class CrawlerAgent implements IAg
   protected IdleCleanupThread idleCleanupThread = null;
   protected SetPriorityThread setPriorityThread = null;
   protected HistoryCleanupThread historyCleanupThread = null;
-
+  protected AssessmentThread assessmentThread = null;
+  
   // Reset managers
   /** Worker thread pool reset manager */
   protected WorkerResetManager workerResetManager = null;
@@ -282,6 +283,55 @@ public class CrawlerAgent implements IAg
     jobManager.noteOutputConnectionChange(connectionName);
   }
 
+  /** Request permission from agent to delete a transformation connection.
+  *@param connName is the name of the transformation connection.
+  *@return true if the connection is in use, false otherwise.
+  */
+  @Override
+  public boolean isTransformationConnectionInUse(IThreadContext threadContext, String connName)
+    throws ManifoldCFException
+  {
+    // Check with job manager.
+    IJobManager jobManager = JobManagerFactory.make(threadContext);
+    return jobManager.checkIfTransformationReference(connName);
+  }
+
+  /** Note the deregistration of a set of transformation connections.
+  *@param connectionNames are the names of the connections being deregistered.
+  */
+  @Override
+  public void noteTransformationConnectorDeregistration(IThreadContext threadContext, String[] connectionNames)
+    throws ManifoldCFException
+  {
+    // Notify job manager
+    IJobManager jobManager = JobManagerFactory.make(threadContext);
+    jobManager.noteTransformationConnectorDeregistration(connectionNames);
+  }
+
+  /** Note the registration of a set of transformation connections.
+  *@param connectionNames are the names of the connections being registered.
+  */
+  @Override
+  public void noteTransformationConnectorRegistration(IThreadContext threadContext, String[] connectionNames)
+    throws ManifoldCFException
+  {
+    // Notify job manager
+    IJobManager jobManager = JobManagerFactory.make(threadContext);
+    jobManager.noteTransformationConnectorRegistration(connectionNames);
+  }
+
+  /** Note a change in configuration for a transformation connection.
+  *@param connectionName is the name of the connection being changed.
+  */
+  @Override
+  public void noteTransformationConnectionChange(IThreadContext threadContext, String connectionName)
+    throws ManifoldCFException
+  {
+    // Notify job manager
+    IJobManager jobManager = JobManagerFactory.make(threadContext);
+    jobManager.noteTransformationConnectionChange(connectionName);
+  }
+
   /** Start everything.
   */
   public void startSystem(IThreadContext threadContext)
@@ -373,6 +423,7 @@ public class CrawlerAgent implements IAg
     jobResetThread = new JobResetThread(processID);
     seedingThread = new SeedingThread(new SeedingResetManager(processID),processID);
     idleCleanupThread = new IdleCleanupThread(processID);
+    assessmentThread = new AssessmentThread(processID);
 
     // Start all the threads
     jobStartThread.start();
@@ -419,6 +470,7 @@ public class CrawlerAgent implements IAg
     jobResetThread.start();
     seedingThread.start();
     idleCleanupThread.start();
+    assessmentThread.start();
 
     Logging.root.info("Pull-agent started");
   }
@@ -434,7 +486,7 @@ public class CrawlerAgent implements IAg
       finisherThread != null || notificationThread != null || workerThreads != null || expireStufferThread != null || expireThreads != null ||
       deleteStufferThread != null || deleteThreads != null ||
       cleanupStufferThread != null || cleanupThreads != null ||
-      jobResetThread != null || seedingThread != null || idleCleanupThread != null || setPriorityThread != null || historyCleanupThread != null)
+      jobResetThread != null || seedingThread != null || idleCleanupThread != null || assessmentThread != null || setPriorityThread != null || historyCleanupThread != null)
     {
       // Send an interrupt to all threads that are still there.
       // In theory, this only needs to be done once.  In practice, I have seen cases where the thread loses track of the fact that it has been
@@ -539,6 +591,10 @@ public class CrawlerAgent implements IAg
       {
         idleCleanupThread.interrupt();
       }
+      if (assessmentThread != null)
+      {
+        assessmentThread.interrupt();
+      }
 
       // Now, wait for all threads to die.
       try
@@ -704,6 +760,11 @@ public class CrawlerAgent implements IAg
         if (!idleCleanupThread.isAlive())
           idleCleanupThread = null;
       }
+      if (assessmentThread != null)
+      {
+        if (!assessmentThread.isAlive())
+          assessmentThread = null;
+      }
     }
 
     // Threads are down; release connectors

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1601529&r1=1601528&r2=1601529&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java Mon Jun  9 23:19:08 2014
@@ -196,6 +196,7 @@ public class ManifoldCF extends org.apac
   // Connectors configuration file
   protected static final String NODE_AUTHORIZATIONDOMAIN = "authorizationdomain";
   protected static final String NODE_OUTPUTCONNECTOR = "outputconnector";
+  protected static final String NODE_TRANSFORMATIONCONNECTOR = "transformationconnector";
   protected static final String NODE_MAPPINGCONNECTOR = "mappingconnector";
   protected static final String NODE_AUTHORITYCONNECTOR = "authorityconnector";
   protected static final String NODE_REPOSITORYCONNECTOR = "repositoryconnector";
@@ -211,6 +212,7 @@ public class ManifoldCF extends org.apac
     // Create a map of class name and description, so we can compare what we can find
     // against what we want.
     Map<String,String> desiredOutputConnectors = new HashMap<String,String>();
+    Map<String,String> desiredTransformationConnectors = new HashMap<String,String>();
     Map<String,String> desiredMappingConnectors = new HashMap<String,String>();
     Map<String,String> desiredAuthorityConnectors = new HashMap<String,String>();
     Map<String,String> desiredRepositoryConnectors = new HashMap<String,String>();
@@ -234,6 +236,12 @@ public class ManifoldCF extends org.apac
           String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
           desiredOutputConnectors.put(className,name);
         }
+        else if (cn.getType().equals(NODE_TRANSFORMATIONCONNECTOR))
+        {
+          String name = cn.getAttributeValue(ATTRIBUTE_NAME);
+          String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
+          desiredTransformationConnectors.put(className,name);
+        }
         else if (cn.getType().equals(NODE_MAPPINGCONNECTOR))
         {
           String name = cn.getAttributeValue(ATTRIBUTE_NAME);
@@ -321,6 +329,49 @@ public class ManifoldCF extends org.apac
       System.err.println("Successfully unregistered all output connectors");
     }
 
+    // Output connectors...
+    {
+      ITransformationConnectorManager mgr = TransformationConnectorManagerFactory.make(tc);
+      ITransformationConnectionManager connManager = TransformationConnectionManagerFactory.make(tc);
+      IResultSet classNames = mgr.getConnectors();
+      int i = 0;
+      while (i < classNames.getRowCount())
+      {
+        IResultRow row = classNames.getRow(i++);
+        String className = (String)row.getValue("classname");
+        String description = (String)row.getValue("description");
+        if (desiredTransformationConnectors.get(className) == null || !desiredTransformationConnectors.get(className).equals(description))
+        {
+          // Deregistration should be done in a transaction
+          database.beginTransaction();
+          try
+          {
+            // Find the connection names that come with this class
+            String[] connectionNames = connManager.findConnectionsForConnector(className);
+            // For all connection names, notify all agents of the deregistration
+            AgentManagerFactory.noteTransformationConnectorDeregistration(tc,connectionNames);
+            // Now that all jobs have been placed into an appropriate state, actually do the deregistration itself.
+            mgr.unregisterConnector(className);
+          }
+          catch (ManifoldCFException e)
+          {
+            database.signalRollback();
+            throw e;
+          }
+          catch (Error e)
+          {
+            database.signalRollback();
+            throw e;
+          }
+          finally
+          {
+            database.endTransaction();
+          }
+        }
+      }
+      System.err.println("Successfully unregistered all transformation connectors");
+    }
+
     // Mapping connectors...
     {
       IMappingConnectorManager mgr = MappingConnectorManagerFactory.make(tc);
@@ -462,6 +513,40 @@ public class ManifoldCF extends org.apac
           }
           System.err.println("Successfully registered output connector '"+className+"'");
         }
+        else if (cn.getType().equals(NODE_TRANSFORMATIONCONNECTOR))
+        {
+          String name = cn.getAttributeValue(ATTRIBUTE_NAME);
+          String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
+          ITransformationConnectorManager mgr = TransformationConnectorManagerFactory.make(tc);
+          ITransformationConnectionManager connManager = TransformationConnectionManagerFactory.make(tc);
+          // Registration should be done in a transaction
+          database.beginTransaction();
+          try
+          {
+            // First, register connector
+            mgr.registerConnector(name,className);
+            // Then, signal to all jobs that might depend on this connector that they can switch state
+            // Find the connection names that come with this class
+            String[] connectionNames = connManager.findConnectionsForConnector(className);
+            // For all connection names, notify all agents of the registration
+            AgentManagerFactory.noteTransformationConnectorRegistration(tc,connectionNames);
+          }
+          catch (ManifoldCFException e)
+          {
+            database.signalRollback();
+            throw e;
+          }
+          catch (Error e)
+          {
+            database.signalRollback();
+            throw e;
+          }
+          finally
+          {
+            database.endTransaction();
+          }
+          System.err.println("Successfully registered transformation connector '"+className+"'");
+        }
         else if (cn.getType().equals(NODE_AUTHORITYCONNECTOR))
         {
           String name = cn.getAttributeValue(ATTRIBUTE_NAME);
@@ -565,6 +650,7 @@ public class ManifoldCF extends org.apac
       ManifoldCF.getMasterDatabasePassword());
     // Also create the following managers, which will handle the actual details of writing configuration data
     IOutputConnectionManager outputManager = OutputConnectionManagerFactory.make(threadContext);
+    ITransformationConnectionManager transManager = TransformationConnectionManagerFactory.make(threadContext);
     IAuthorityGroupManager groupManager = AuthorityGroupManagerFactory.make(threadContext);
     IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(threadContext);
     IMappingConnectionManager mappingManager = MappingConnectionManagerFactory.make(threadContext);
@@ -624,6 +710,11 @@ public class ManifoldCF extends org.apac
 
             // The zipentries must be written in an order that permits their proper restoration.  The "lowest level" is thus
             // written first, which yields the order: authority connections, repository connections, jobs
+            java.util.zip.ZipEntry transEntry = new java.util.zip.ZipEntry("transformations");
+            zos.putNextEntry(transEntry);
+            transManager.exportConfiguration(zos);
+            zos.closeEntry();
+
             java.util.zip.ZipEntry outputEntry = new java.util.zip.ZipEntry("outputs");
             zos.putNextEntry(outputEntry);
             outputManager.exportConfiguration(zos);
@@ -705,6 +796,7 @@ public class ManifoldCF extends org.apac
       ManifoldCF.getMasterDatabasePassword());
     // Also create the following managers, which will handle the actual details of reading configuration data
     IOutputConnectionManager outputManager = OutputConnectionManagerFactory.make(threadContext);
+    ITransformationConnectionManager transManager = TransformationConnectionManagerFactory.make(threadContext);
     IAuthorityGroupManager groupManager = AuthorityGroupManagerFactory.make(threadContext);
     IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(threadContext);
     IMappingConnectionManager mappingManager = MappingConnectionManagerFactory.make(threadContext);
@@ -761,7 +853,9 @@ public class ManifoldCF extends org.apac
               entries++;
               // Get the name of the entry
               String name = z.getName();
-              if (name.equals("outputs"))
+              if (name.equals("transformations"))
+                transManager.importConfiguration(zis);
+              else if (name.equals("outputs"))
                 outputManager.importConfiguration(zis);
               else if (name.equals("groups"))
                 groupManager.importConfiguration(zis);
@@ -1029,16 +1123,6 @@ public class ManifoldCF extends org.apac
 
   }
 
-  /** Qualify output activity name.
-  *@param outputActivityName is the name of the output activity.
-  *@param outputConnectionName is the corresponding name of the output connection.
-  *@return the qualified (global) activity name.
-  */
-  public static String qualifyOutputActivityName(String outputActivityName, String outputConnectionName)
-  {
-    return outputActivityName+" ("+outputConnectionName+")";
-  }
-
   /** Get the activities list for a given repository connection.
   */
   public static String[] getActivitiesList(IThreadContext threadContext, String connectionName)
@@ -1049,31 +1133,35 @@ public class ManifoldCF extends org.apac
     if (thisConnection == null)
       return null;
     String[] outputActivityList = OutputConnectionManagerFactory.getAllOutputActivities(threadContext);
+    String[] transformationActivityList = TransformationConnectionManagerFactory.getAllTransformationActivities(threadContext);
     String[] connectorActivityList = RepositoryConnectorFactory.getActivitiesList(threadContext,thisConnection.getClassName());
     String[] globalActivityList = IRepositoryConnectionManager.activitySet;
-    String[] activityList = new String[outputActivityList.length + ((connectorActivityList==null)?0:connectorActivityList.length) + globalActivityList.length];
+    String[] activityList = new String[transformationActivityList.length + outputActivityList.length + ((connectorActivityList==null)?0:connectorActivityList.length) + globalActivityList.length];
     int k2 = 0;
-    int j;
+    if (transformationActivityList != null)
+    {
+      for (String transformationActivity: transformationActivityList)
+      {
+        activityList[k2++] = transformationActivity;
+      }
+    }
     if (outputActivityList != null)
     {
-      j = 0;
-      while (j < outputActivityList.length)
+      for (String outputActivity : outputActivityList)
       {
-        activityList[k2++] = outputActivityList[j++];
+        activityList[k2++] = outputActivity;
       }
     }
     if (connectorActivityList != null)
     {
-      j = 0;
-      while (j < connectorActivityList.length)
+      for (String connectorActivity : connectorActivityList)
       {
-        activityList[k2++] = connectorActivityList[j++];
+        activityList[k2++] = connectorActivity;
       }
     }
-    j = 0;
-    while (j < globalActivityList.length)
+    for (String globalActivity : globalActivityList)
     {
-      activityList[k2++] = globalActivityList[j++];
+      activityList[k2++] = globalActivity;
     }
     java.util.Arrays.sort(activityList);
     return activityList;
@@ -1109,10 +1197,12 @@ public class ManifoldCF extends org.apac
   protected static final String API_AUTHORITYGROUPNODE = "authoritygroup";
   protected static final String API_REPOSITORYCONNECTORNODE = "repositoryconnector";
   protected static final String API_OUTPUTCONNECTORNODE = "outputconnector";
+  protected static final String API_TRANSFORMATIONCONNECTORNODE = "transformationconnector";
   protected static final String API_AUTHORITYCONNECTORNODE = "authorityconnector";
   protected static final String API_MAPPINGCONNECTORNODE = "mappingconnector";
   protected static final String API_REPOSITORYCONNECTIONNODE = "repositoryconnection";
   protected static final String API_OUTPUTCONNECTIONNODE = "outputconnection";
+  protected static final String API_TRANSFORMATIONCONNECTIONNODE = "transformationconnection";
   protected static final String API_AUTHORITYCONNECTIONNODE = "authorityconnection";
   protected static final String API_MAPPINGCONNECTIONNODE = "mappingconnection";
   protected static final String API_CHECKRESULTNODE = "check_result";
@@ -1262,6 +1352,48 @@ public class ManifoldCF extends org.apac
     return READRESULT_FOUND;
   }
 
+  /** Read a transformation connection status */
+  protected static int apiReadTransformationConnectionStatus(IThreadContext tc, Configuration output, String connectionName)
+    throws ManifoldCFException
+  {
+    try
+    {
+      ITransformationConnectorPool transformationConnectorPool = TransformationConnectorPoolFactory.make(tc);
+      ITransformationConnectionManager connectionManager = TransformationConnectionManagerFactory.make(tc);
+      ITransformationConnection connection = connectionManager.load(connectionName);
+      if (connection == null)
+      {
+        createErrorNode(output,"Connection '"+connectionName+"' does not exist");
+        return READRESULT_NOTFOUND;
+      }
+          
+      String results;
+      // Grab a connection handle, and call the test method
+      ITransformationConnector connector = transformationConnectorPool.grab(connection);
+      try
+      {
+        results = connector.check();
+      }
+      catch (ManifoldCFException e)
+      {
+        results = e.getMessage();
+      }
+      finally
+      {
+        transformationConnectorPool.release(connection,connector);
+      }
+          
+      ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE);
+      response.setValue(results);
+      output.addChild(output.getChildCount(),response);
+    }
+    catch (ManifoldCFException e)
+    {
+      createErrorNode(output,e);
+    }
+    return READRESULT_FOUND;
+  }
+
   /** Read an authority connection status */
   protected static int apiReadAuthorityConnectionStatus(IThreadContext tc, Configuration output, String connectionName)
     throws ManifoldCFException
@@ -1421,6 +1553,39 @@ public class ManifoldCF extends org.apac
     }
     return READRESULT_FOUND;
   }
+
+  /** Read a transformation connection's info */
+  protected static int apiReadTransformationConnectionInfo(IThreadContext tc, Configuration output, String connectionName, String command)
+    throws ManifoldCFException
+  {
+    try
+    {
+      ITransformationConnectorPool transformationConnectorPool = TransformationConnectorPoolFactory.make(tc);
+      ITransformationConnectionManager connectionManager = TransformationConnectionManagerFactory.make(tc);
+      ITransformationConnection connection = connectionManager.load(connectionName);
+      if (connection == null)
+      {
+        createErrorNode(output,"Connection '"+connectionName+"' does not exist");
+        return READRESULT_NOTFOUND;
+      }
+
+      // Grab a connection handle, and call the test method
+      ITransformationConnector connector = transformationConnectorPool.grab(connection);
+      try
+      {
+        return connector.requestInfo(output,command)?READRESULT_FOUND:READRESULT_NOTFOUND;
+      }
+      finally
+      {
+        transformationConnectorPool.release(connection,connector);
+      }
+    }
+    catch (ManifoldCFException e)
+    {
+      createErrorNode(output,e);
+    }
+    return READRESULT_FOUND;
+  }
   
   /** Read a repository connection's info */
   protected static int apiReadRepositoryConnectionInfo(IThreadContext tc, Configuration output, String connectionName, String command)
@@ -1669,6 +1834,57 @@ public class ManifoldCF extends org.apac
     return READRESULT_FOUND;
   }
 
+    /** Get transformation connections */
+  protected static int apiReadTransformationConnections(IThreadContext tc, Configuration output)
+    throws ManifoldCFException
+  {
+    try
+    {
+      ITransformationConnectionManager connManager = TransformationConnectionManagerFactory.make(tc);
+      ITransformationConnection[] connections = connManager.getAllConnections();
+      int i = 0;
+      while (i < connections.length)
+      {
+        ConfigurationNode connectionNode = new ConfigurationNode(API_TRANSFORMATIONCONNECTIONNODE);
+        formatTransformationConnection(connectionNode,connections[i++]);
+        output.addChild(output.getChildCount(),connectionNode);
+      }
+    }
+    catch (ManifoldCFException e)
+    {
+      createErrorNode(output,e);
+    }
+    return READRESULT_FOUND;
+  }
+  
+  /** Read transformation connection */
+  protected static int apiReadTransformationConnection(IThreadContext tc, Configuration output, String connectionName)
+    throws ManifoldCFException
+  {
+    try
+    {
+      ITransformationConnectionManager connectionManager = TransformationConnectionManagerFactory.make(tc);
+      ITransformationConnection connection = connectionManager.load(connectionName);
+      if (connection != null)
+      {
+        // Fill the return object with job information
+        ConfigurationNode connectionNode = new ConfigurationNode(API_TRANSFORMATIONCONNECTIONNODE);
+        formatTransformationConnection(connectionNode,connection);
+        output.addChild(output.getChildCount(),connectionNode);
+      }
+      else
+      {
+        createErrorNode(output,"Connection '"+connectionName+"' does not exist.");
+        return READRESULT_NOTFOUND;
+      }
+    }
+    catch (ManifoldCFException e)
+    {
+      createErrorNode(output,e);
+    }
+    return READRESULT_FOUND;
+  }
+
   /** Get authority connections */
   protected static int apiReadAuthorityConnections(IThreadContext tc, Configuration output)
     throws ManifoldCFException
@@ -1859,6 +2075,43 @@ public class ManifoldCF extends org.apac
     return READRESULT_FOUND;
   }
 
+  /** List transformation connectors */
+  protected static int apiReadTransformationConnectors(IThreadContext tc, Configuration output)
+    throws ManifoldCFException
+  {
+    // List registered transformation connectors
+    try
+    {
+      ITransformationConnectorManager manager = TransformationConnectorManagerFactory.make(tc);
+      IResultSet resultSet = manager.getConnectors();
+      int j = 0;
+      while (j < resultSet.getRowCount())
+      {
+        IResultRow row = resultSet.getRow(j++);
+        ConfigurationNode child = new ConfigurationNode(API_TRANSFORMATIONCONNECTORNODE);
+        String description = (String)row.getValue("description");
+        String className = (String)row.getValue("classname");
+        ConfigurationNode node;
+        if (description != null)
+        {
+          node = new ConfigurationNode(CONNECTORNODE_DESCRIPTION);
+          node.setValue(description);
+          child.addChild(child.getChildCount(),node);
+        }
+        node = new ConfigurationNode(CONNECTORNODE_CLASSNAME);
+        node.setValue(className);
+        child.addChild(child.getChildCount(),node);
+
+        output.addChild(output.getChildCount(),child);
+      }
+    }
+    catch (ManifoldCFException e)
+    {
+      createErrorNode(output,e);
+    }
+    return READRESULT_FOUND;
+  }
+
   /** List authority connectors */
   protected static int apiReadAuthorityConnectors(IThreadContext tc, Configuration output)
     throws ManifoldCFException
@@ -2681,6 +2934,10 @@ public class ManifoldCF extends org.apac
       {
         return apiReadOutputConnectionStatus(tc,output,connectionName);
       }
+      else if (connectionType.equals("transformationconnections"))
+      {
+        return apiReadTransformationConnectionStatus(tc,output,connectionName);
+      }
       else if (connectionType.equals("mappingconnections"))
       {
         return apiReadMappingConnectionStatus(tc,output,connectionName);
@@ -2724,6 +2981,10 @@ public class ManifoldCF extends org.apac
       {
         return apiReadOutputConnectionInfo(tc,output,connectionName,command);
       }
+      else if (connectionType.equals("transformationconnections"))
+      {
+        return apiReadTransformationConnectionInfo(tc,output,connectionName,command);
+      }
       else if (connectionType.equals("repositoryconnections"))
       {
         return apiReadRepositoryConnectionInfo(tc,output,connectionName,command);
@@ -2770,6 +3031,15 @@ public class ManifoldCF extends org.apac
       String connectionName = decodeAPIPathElement(path.substring("outputconnections/".length()));
       return apiReadOutputConnection(tc,output,connectionName);
     }
+    else if (path.equals("transformationconnections"))
+    {
+      return apiReadTransformationConnections(tc,output);
+    }
+    else if (path.startsWith("transformationconnections/"))
+    {
+      String connectionName = decodeAPIPathElement(path.substring("transformationconnections/".length()));
+      return apiReadTransformationConnection(tc,output,connectionName);
+    }
     else if (path.equals("mappingconnections"))
     {
       return apiReadMappingConnections(tc,output);
@@ -2801,6 +3071,10 @@ public class ManifoldCF extends org.apac
     {
       return apiReadOutputConnectors(tc,output);
     }
+    else if (path.equals("transformationconnectors"))
+    {
+      return apiReadTransformationConnectors(tc,output);
+    }
     else if (path.equals("mappingconnectors"))
     {
       return apiReadMappingConnectors(tc,output);
@@ -3109,6 +3383,41 @@ public class ManifoldCF extends org.apac
     }
     return WRITERESULT_FOUND;
   }
+
+  /** Write transformation connection.
+  */
+  protected static int apiWriteTransformationConnection(IThreadContext tc, Configuration output, Configuration input, String connectionName)
+    throws ManifoldCFException
+  {
+    ConfigurationNode connectionNode = findConfigurationNode(input,API_TRANSFORMATIONCONNECTIONNODE);
+    if (connectionNode == null)
+      throw new ManifoldCFException("Input argument must have '"+API_TRANSFORMATIONCONNECTIONNODE+"' field");
+      
+    // Turn the configuration node into a TransformationConnection
+    org.apache.manifoldcf.agents.transformationconnection.TransformationConnection transformationConnection = new org.apache.manifoldcf.agents.transformationconnection.TransformationConnection();
+    processTransformationConnection(transformationConnection,connectionNode);
+      
+    if (transformationConnection.getName() == null)
+      transformationConnection.setName(connectionName);
+    else
+    {
+      if (!transformationConnection.getName().equals(connectionName))
+        throw new ManifoldCFException("Connection name in path and in object must agree");
+    }
+      
+    try
+    {
+      // Save the connection.
+      ITransformationConnectionManager connectionManager = TransformationConnectionManagerFactory.make(tc);
+      if (connectionManager.save(transformationConnection))
+        return WRITERESULT_CREATED;
+    }
+    catch (ManifoldCFException e)
+    {
+      createErrorNode(output,e);
+    }
+    return WRITERESULT_FOUND;
+  }
   
   /** Write authority connection.
   */
@@ -3332,6 +3641,11 @@ public class ManifoldCF extends org.apac
       String connectionName = decodeAPIPathElement(path.substring("outputconnections/".length()));
       return apiWriteOutputConnection(tc,output,input,connectionName);
     }
+    else if (path.startsWith("transformationconnections/"))
+    {
+      String connectionName = decodeAPIPathElement(path.substring("transformationconnections/".length()));
+      return apiWriteTransformationConnection(tc,output,input,connectionName);
+    }
     else if (path.startsWith("mappingconnections/"))
     {
       String connectionName = decodeAPIPathElement(path.substring("mappingconnections/".length()));
@@ -3560,6 +3874,10 @@ public class ManifoldCF extends org.apac
   protected static final String JOBNODE_FORCEDPARAM = "forcedparam";
   protected static final String JOBNODE_PARAMNAME = "paramname";
   protected static final String JOBNODE_PARAMVALUE = "paramvalue";
+  protected static final String JOBNODE_PIPELINESTAGE = "pipelinestage";
+  protected static final String JOBNODE_STAGECONNECTIONNAME = "stage_connectionname";
+  protected static final String JOBNODE_STAGEDESCRIPTION = "stage_description";
+  protected static final String JOBNODE_STAGESPECIFICATION = "stage_specification";
 
   /** Convert a node into a job description.
   *@param jobDescription is the job to be filled in.
@@ -3592,15 +3910,43 @@ public class ManifoldCF extends org.apac
       {
         jobDescription.setOutputConnectionName(child.getValue());
       }
+      else if (childType.equals(JOBNODE_PIPELINESTAGE))
+      {
+        String stageConnectionName = null;
+        String stageDescription = null;
+        ConfigurationNode stageSpecification = null;
+        for (int q = 0; q < child.getChildCount(); q++)
+        {
+          ConfigurationNode cn = child.findChild(q);
+          if (cn.getType().equals(JOBNODE_STAGECONNECTIONNAME))
+            stageConnectionName = cn.getValue();
+          else if (cn.getType().equals(JOBNODE_STAGEDESCRIPTION))
+            stageDescription = cn.getValue();
+          else if (cn.getType().equals(JOBNODE_STAGESPECIFICATION))
+          {
+            stageSpecification = cn;
+          }
+          else
+            throw new ManifoldCFException("Found an unexpected node type: '"+cn.getType()+"'");
+        }
+        if (stageConnectionName == null)
+          throw new ManifoldCFException("Missing required field: '"+JOBNODE_STAGECONNECTIONNAME+"'");
+        OutputSpecification os = jobDescription.addPipelineStage(stageConnectionName,stageDescription);
+        os.clearChildren();
+        for (int j = 0; j < stageSpecification.getChildCount(); j++)
+        {
+          ConfigurationNode cn = stageSpecification.findChild(j);
+          os.addChild(os.getChildCount(),new SpecificationNode(cn));
+        }
+      }
       else if (childType.equals(JOBNODE_DOCUMENTSPECIFICATION))
       {
         // Get the job's document specification, clear out the children, and copy new ones from the child.
         DocumentSpecification ds = jobDescription.getSpecification();
         ds.clearChildren();
-        int j = 0;
-        while (j < child.getChildCount())
+        for (int j = 0; j < child.getChildCount(); j++)
         {
-          ConfigurationNode cn = child.findChild(j++);
+          ConfigurationNode cn = child.findChild(j);
           ds.addChild(ds.getChildCount(),new SpecificationNode(cn));
         }
       }
@@ -3609,10 +3955,9 @@ public class ManifoldCF extends org.apac
         // Get the job's output specification, clear out the children, and copy new ones from the child.
         OutputSpecification os = jobDescription.getOutputSpecification();
         os.clearChildren();
-        int j = 0;
-        while (j < child.getChildCount())
+        for (int j = 0; j < child.getChildCount(); j++)
         {
-          ConfigurationNode cn = child.findChild(j++);
+          ConfigurationNode cn = child.findChild(j);
           os.addChild(os.getChildCount(),new SpecificationNode(cn));
         }
       }
@@ -3770,7 +4115,6 @@ public class ManifoldCF extends org.apac
   {
     // For each field of the job, add an appropriate child node, with value.
     ConfigurationNode child;
-    int j;
     
     // id
     if (job.getID() != null)
@@ -3807,10 +4151,9 @@ public class ManifoldCF extends org.apac
     // Document specification
     DocumentSpecification ds = job.getSpecification();
     child = new ConfigurationNode(JOBNODE_DOCUMENTSPECIFICATION);
-    j = 0;
-    while (j < ds.getChildCount())
+    for (int j = 0; j < ds.getChildCount(); j++)
     {
-      ConfigurationNode cn = ds.getChild(j++);
+      ConfigurationNode cn = ds.getChild(j);
       child.addChild(child.getChildCount(),cn);
     }
     jobNode.addChild(jobNode.getChildCount(),child);
@@ -3818,14 +4161,39 @@ public class ManifoldCF extends org.apac
     // Output specification
     OutputSpecification os = job.getOutputSpecification();
     child = new ConfigurationNode(JOBNODE_OUTPUTSPECIFICATION);
-    j = 0;
-    while (j < os.getChildCount())
+    for (int j = 0; j < os.getChildCount(); j++)
     {
-      ConfigurationNode cn = os.getChild(j++);
+      ConfigurationNode cn = os.getChild(j);
       child.addChild(child.getChildCount(),cn);
     }
     jobNode.addChild(jobNode.getChildCount(),child);
 
+    // Pipeline stages
+    for (int j = 0; j < job.countPipelineStages(); j++)
+    {
+      child = new ConfigurationNode(JOBNODE_PIPELINESTAGE);
+      ConfigurationNode stage;
+      stage = new ConfigurationNode(JOBNODE_STAGECONNECTIONNAME);
+      stage.setValue(job.getPipelineStageConnectionName(j));
+      child.addChild(child.getChildCount(),stage);
+      String description = job.getPipelineStageDescription(j);
+      if (description != null)
+      {
+        stage = new ConfigurationNode(JOBNODE_STAGEDESCRIPTION);
+        stage.setValue(description);
+        child.addChild(child.getChildCount(),stage);
+      }
+      OutputSpecification spec = job.getPipelineStageSpecification(j);
+      stage = new ConfigurationNode(JOBNODE_STAGESPECIFICATION);
+      for (int k = 0; k < spec.getChildCount(); k++)
+      {
+        ConfigurationNode cn = spec.getChild(k);
+        stage.addChild(stage.getChildCount(),cn);
+      }
+      child.addChild(child.getChildCount(),stage);
+      jobNode.addChild(jobNode.getChildCount(),child);
+    }
+
     // Start mode
     child = new ConfigurationNode(JOBNODE_STARTMODE);
     child.setValue(startModeMap(job.getStartMethod()));
@@ -3897,10 +4265,9 @@ public class ManifoldCF extends org.apac
     }
     
     // Schedule records
-    j = 0;
-    while (j < job.getScheduleRecordCount())
+    for (int j = 0; j < job.getScheduleRecordCount(); j++)
     {
-      ScheduleRecord sr = job.getScheduleRecord(j++);
+      ScheduleRecord sr = job.getScheduleRecord(j);
       child = new ConfigurationNode(JOBNODE_SCHEDULE);
       ConfigurationNode recordChild;
       
@@ -4406,6 +4773,118 @@ public class ManifoldCF extends org.apac
 
   }
 
+  // Transformation connection API support
+  
+    /** Convert input hierarchy into a TransformationConnection object.
+  */
+  protected static void processTransformationConnection(org.apache.manifoldcf.agents.transformationconnection.TransformationConnection connection, ConfigurationNode connectionNode)
+    throws ManifoldCFException
+  {
+    // Walk through the node's children
+    int i = 0;
+    while (i < connectionNode.getChildCount())
+    {
+      ConfigurationNode child = connectionNode.findChild(i++);
+      String childType = child.getType();
+      if (childType.equals(CONNECTIONNODE_ISNEW))
+      {
+        if (child.getValue() == null)
+          throw new ManifoldCFException("Connection isnew node requires a value");
+        connection.setIsNew(child.getValue().equals("true"));
+      }
+      else if (childType.equals(CONNECTIONNODE_NAME))
+      {
+        if (child.getValue() == null)
+          throw new ManifoldCFException("Connection name node requires a value");
+        connection.setName(child.getValue());
+      }
+      else if (childType.equals(CONNECTIONNODE_CLASSNAME))
+      {
+        if (child.getValue() == null)
+          throw new ManifoldCFException("Connection classname node requires a value");
+        connection.setClassName(child.getValue());
+      }
+      else if (childType.equals(CONNECTIONNODE_MAXCONNECTIONS))
+      {
+        if (child.getValue() == null)
+          throw new ManifoldCFException("Connection maxconnections node requires a value");
+        try
+        {
+          connection.setMaxConnections(Integer.parseInt(child.getValue()));
+        }
+        catch (NumberFormatException e)
+        {
+          throw new ManifoldCFException("Error parsing max connections: "+e.getMessage(),e);
+        }
+      }
+      else if (childType.equals(CONNECTIONNODE_DESCRIPTION))
+      {
+        if (child.getValue() == null)
+          throw new ManifoldCFException("Connection description node requires a value");
+        connection.setDescription(child.getValue());
+      }
+      else if (childType.equals(CONNECTIONNODE_CONFIGURATION))
+      {
+        // Get the connection's configuration, clear out the children, and copy new ones from the child.
+        ConfigParams cp = connection.getConfigParams();
+        cp.clearChildren();
+        int j = 0;
+        while (j < child.getChildCount())
+        {
+          ConfigurationNode cn = child.findChild(j++);
+          cp.addChild(cp.getChildCount(),new ConfigNode(cn));
+        }
+      }
+      else
+        throw new ManifoldCFException("Unrecognized output connection field: '"+childType+"'");
+    }
+    if (connection.getClassName() == null)
+      throw new ManifoldCFException("Missing connection field: '"+CONNECTIONNODE_CLASSNAME+"'");
+
+  }
+
+  /** Format a transformation connection.
+  */
+  protected static void formatTransformationConnection(ConfigurationNode connectionNode, ITransformationConnection connection)
+  {
+    ConfigurationNode child;
+    int j;
+
+    child = new ConfigurationNode(CONNECTIONNODE_ISNEW);
+    child.setValue(connection.getIsNew()?"true":"false");
+    connectionNode.addChild(connectionNode.getChildCount(),child);
+
+    child = new ConfigurationNode(CONNECTIONNODE_NAME);
+    child.setValue(connection.getName());
+    connectionNode.addChild(connectionNode.getChildCount(),child);
+
+    child = new ConfigurationNode(CONNECTIONNODE_CLASSNAME);
+    child.setValue(connection.getClassName());
+    connectionNode.addChild(connectionNode.getChildCount(),child);
+
+    child = new ConfigurationNode(CONNECTIONNODE_MAXCONNECTIONS);
+    child.setValue(Integer.toString(connection.getMaxConnections()));
+    connectionNode.addChild(connectionNode.getChildCount(),child);
+
+    if (connection.getDescription() != null)
+    {
+      child = new ConfigurationNode(CONNECTIONNODE_DESCRIPTION);
+      child.setValue(connection.getDescription());
+      connectionNode.addChild(connectionNode.getChildCount(),child);
+    }
+    
+    ConfigParams cp = connection.getConfigParams();
+    child = new ConfigurationNode(CONNECTIONNODE_CONFIGURATION);
+    j = 0;
+    while (j < cp.getChildCount())
+    {
+      ConfigurationNode cn = cp.findChild(j++);
+      child.addChild(child.getChildCount(),cn);
+    }
+    connectionNode.addChild(connectionNode.getChildCount(),child);
+
+  }
+
   // Authority connection API support
   
   /** Convert input hierarchy into an AuthorityConnection object.

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1601529&r1=1601528&r2=1601529&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java Mon Jun  9 23:19:08 2014
@@ -145,14 +145,22 @@ public class WorkerThread extends Thread
             // Universal data, from the job
             String connectionName = job.getConnectionName();
             String outputName = job.getOutputConnectionName();
-            String newParameterVersion = packParameters(job.getForcedMetadata());
+            int pipelineCount = job.countPipelineStages();
+            String[] transformationNames = new String[pipelineCount];
+            OutputSpecification[] transformationSpecifications = new OutputSpecification[pipelineCount];
+            for (int k = 0; k < pipelineCount; k++)
+            {
+              transformationNames[k] = job.getPipelineStageConnectionName(k);
+              transformationSpecifications[k] = job.getPipelineStageSpecification(k);
+            }
+            
             DocumentSpecification spec = job.getSpecification();
             OutputSpecification outputSpec = job.getOutputSpecification();
             int jobType = job.getType();
 
             IRepositoryConnection connection = qds.getConnection();
             
-            OutputActivity ingestLogger = new OutputActivity(connectionName,connMgr,outputName);
+            OutputActivity ingestLogger = new OutputActivity(connectionName,connMgr);
 
             // The flow through this section of the code is as follows.
             // (1) We start with a list of documents
@@ -307,14 +315,23 @@ public class WorkerThread extends Thread
                       }
                     }
 
-                    // Get the output version string.
-                    String outputVersion = ingester.getOutputDescription(outputName,outputSpec);
-                      
-                    HashMap abortSet = new HashMap();
-                    VersionActivity versionActivity = new VersionActivity(processID,connectionName,connMgr,jobManager,job,ingester,abortSet,outputVersion);
+                    // Get the output version string. Cannot be null.
+                    String outputDescriptionString = ingester.getOutputDescription(outputName,outputSpec);
+                    // Get the transformation version strings.  Cannot be null.
+                    String[] transformationDescriptionStrings = ingester.getTransformationDescriptions(transformationNames,transformationSpecifications);
+                    
+                    // New version strings
+                    String newOutputVersion = outputDescriptionString;
+                    String newParameterVersion = packParameters(job.getForcedMetadata());
+                    String newTransformationVersion = packTransformations(transformationNames,transformationDescriptionStrings);
+
+                    Set<String> abortSet = new HashSet<String>();
+                    VersionActivity versionActivity = new VersionActivity(job.getID(),processID,connectionName,outputName,transformationNames,connMgr,jobManager,ingester,abortSet,outputDescriptionString,transformationDescriptionStrings,ingestLogger);
 
                     String aclAuthority = connection.getACLAuthority();
-                    boolean isDefaultAuthority = (aclAuthority == null || aclAuthority.length() == 0);
+                    if (aclAuthority == null)
+                      aclAuthority = "";
+                    boolean isDefaultAuthority = (aclAuthority.length() == 0);
 
                     if (Logging.threads.isDebugEnabled())
                       Logging.threads.debug("Worker thread getting versions for "+Integer.toString(currentDocIDArray.length)+" documents");
@@ -390,10 +407,6 @@ public class WorkerThread extends Thread
                       // This try{ } is for releasing document versions at the connector level.
                       try
                       {
-                        // Organize what we need for document status comparison, and get it into a canonical form.
-                        String newOutputVersion = outputVersion;
-                        if (newOutputVersion == null)
-                          newOutputVersion = "";
 
                         // Loop through documents now, and amass what we need to fetch.
                         // We also need to tally: (1) what needs to be marked as deleted via
@@ -405,7 +418,7 @@ public class WorkerThread extends Thread
                           QueuedDocument qd = activeDocuments.get(i);
                           DocumentDescription dd = qd.getDocumentDescription();
                           // If this document was aborted, then treat it specially; we never go on to fetch it, for one thing.
-                          if (abortSet.get(dd.getDocumentIdentifier()) != null)
+                          if (abortSet.contains(dd.getDocumentIdentifier()))
                           {
                             // Special treatment for aborted documents.
                             // We ignore the returned version string completely, since it's presumed that processing was not completed for this doc.
@@ -419,9 +432,6 @@ public class WorkerThread extends Thread
                             DocumentIngestStatus oldDocStatus = qd.getLastIngestedStatus();
                             String documentIDHash = dd.getDocumentIdentifierHash();
                             String newDocVersion = newVersionStringArray[i];
-                            String newAuthorityName = aclAuthority;
-                            if (newAuthorityName == null)
-                              newAuthorityName = "";
 
                             versionMap.put(dd.getDocumentIdentifierHash(),newDocVersion);
 
@@ -448,17 +458,10 @@ public class WorkerThread extends Thread
                                 // that was there before is there now (which may mean a rescan),
                                 // or (2) there are different versions (which ALWAYS means a rescan).
                                 String oldDocVersion = oldDocStatus.getDocumentVersion();
-                                if (oldDocVersion == null)
-                                  oldDocVersion = "";
                                 String oldAuthorityName = oldDocStatus.getDocumentAuthorityNameString();
-                                if (oldAuthorityName == null)
-                                  oldAuthorityName = "";
                                 String oldOutputVersion = oldDocStatus.getOutputVersion();
-                                if (oldOutputVersion == null)
-                                  oldOutputVersion = "";
+                                String oldTransformationVersion = oldDocStatus.getTransformationVersion();
                                 String oldParameterVersion = oldDocStatus.getParameterVersion();
-                                if (oldParameterVersion == null)
-                                  oldParameterVersion = "";
 
                                 // Start the comparison processing
                                 if (newDocVersion.length() == 0)
@@ -467,8 +470,9 @@ public class WorkerThread extends Thread
                                   allowIngest = true;
                                 }
                                 else if (oldDocVersion.equals(newDocVersion) &&
-                                  oldAuthorityName.equals(newAuthorityName) &&
+                                  oldAuthorityName.equals(aclAuthority) &&
                                   oldOutputVersion.equals(newOutputVersion) &&
+                                  oldTransformationVersion.equals(newTransformationVersion) &&
                                   oldParameterVersion.equals(newParameterVersion))
                                 {
                                   // The old logic was as follows:
@@ -527,9 +531,18 @@ public class WorkerThread extends Thread
                         }
 
                         // First, make the things we will need for all subsequent steps.
-                        ProcessActivity activity = new ProcessActivity(processID,
+                        ProcessActivity activity = new ProcessActivity(job.getID(),processID,
                           threadContext,rt,jobManager,ingester,
-                          currentTime,job,connection,connector,connMgr,legalLinkTypes,ingestLogger,abortSet,outputVersion,newParameterVersion);
+                          connectionName,outputName,transformationNames,
+                          outputDescriptionString,transformationDescriptionStrings,
+                          currentTime,
+                          job.getExpiration(),
+                          job.getForcedMetadata(),
+                          job.getInterval(),
+                          job.getMaxInterval(),
+                          job.getHopcountMode(),
+                          connection,connector,connMgr,legalLinkTypes,ingestLogger,abortSet,
+                          newOutputVersion,newTransformationVersion,newParameterVersion);
                         try
                         {
 
@@ -677,7 +690,7 @@ public class WorkerThread extends Thread
                                   String documentID = recrawlDocs[i].getDocumentIdentifier();
 
                                   // If aborted due to sequencing issue, then requeue for reprocessing immediately, ignoring everything else.
-                                  boolean wasAborted = abortSet.get(documentID) != null;
+                                  boolean wasAborted = abortSet.contains(documentID);
                                   if (wasAborted)
                                   {
                                     // Requeue for immediate reprocessing
@@ -742,7 +755,7 @@ public class WorkerThread extends Thread
                                 {
                                   QueuedDocument qd = finishList.get(i);
                                   DocumentDescription dd = qd.getDocumentDescription();
-                                  if (abortSet.get(dd.getDocumentIdentifier()) != null)
+                                  if (abortSet.contains(dd.getDocumentIdentifier()))
                                   {
                                     // The document was aborted, so put it into the abortedList
                                     abortedList.add(dd);
@@ -1186,6 +1199,25 @@ public class WorkerThread extends Thread
     }
   }
 
+  protected static String packTransformations(String[] transformationNames, String[] transformationDescriptionStrings)
+  {
+    StringBuilder sb = new StringBuilder();
+    packList(sb,transformationNames,'+');
+    packList(sb,transformationDescriptionStrings,'!');
+    return sb.toString();
+  }
+  
+  /** Another stuffer for packing lists of variable length */
+  protected static void packList(StringBuilder output, String[] values, char delimiter)
+  {
+    pack(output,Integer.toString(values.length),delimiter);
+    int i = 0;
+    while (i < values.length)
+    {
+      pack(output,values[i++],delimiter);
+    }
+  }
+
   protected static String packParameters(Map<String,Set<String>> forcedParameters)
   {
     StringBuilder sb = new StringBuilder();
@@ -1238,59 +1270,82 @@ public class WorkerThread extends Thread
   */
   protected static class VersionActivity implements IVersionActivity
   {
+    protected final Long jobID;
     protected final String processID;
     protected final String connectionName;
+    protected final String outputConnectionName;
+    protected final String[] transformationConnectionNames;
     protected final IRepositoryConnectionManager connMgr;
     protected final IJobManager jobManager;
-    protected final IJobDescription job;
     protected final IIncrementalIngester ingester;
-    protected final HashMap abortSet;
-    protected final String outputVersion;
-
+    protected final Set<String> abortSet;
+    protected final String outputDescriptionString;
+    protected final String[] transformationDescriptionStrings;
+    protected final CheckActivity checkActivity;
     /** Constructor.
     */
-    public VersionActivity(String processID, String connectionName, IRepositoryConnectionManager connMgr,
-      IJobManager jobManager, IJobDescription job, IIncrementalIngester ingester, HashMap abortSet,
-      String outputVersion)
+    public VersionActivity(Long jobID, String processID,
+      String connectionName, String outputConnectionName,
+      String[] transformationConnectionNames,
+      IRepositoryConnectionManager connMgr,
+      IJobManager jobManager, IIncrementalIngester ingester, Set<String> abortSet,
+      String outputDescriptionString, String[] transformationDescriptionStrings,
+      CheckActivity checkActivity)
     {
+      this.jobID = jobID;
       this.processID = processID;
       this.connectionName = connectionName;
+      this.outputConnectionName = outputConnectionName;
+      this.transformationConnectionNames = transformationConnectionNames;
       this.connMgr = connMgr;
       this.jobManager = jobManager;
-      this.job = job;
       this.ingester = ingester;
       this.abortSet = abortSet;
-      this.outputVersion = outputVersion;
+      this.outputDescriptionString = outputDescriptionString;
+      this.transformationDescriptionStrings = transformationDescriptionStrings;
+      this.checkActivity = checkActivity;
     }
 
     /** Check whether a mime type is indexable by the currently specified output connector.
     *@param mimeType is the mime type to check, not including any character set specification.
     *@return true if the mime type is indexable.
     */
+    @Override
     public boolean checkMimeTypeIndexable(String mimeType)
       throws ManifoldCFException, ServiceInterruption
     {
-      return ingester.checkMimeTypeIndexable(job.getOutputConnectionName(),outputVersion,mimeType);
+      return ingester.checkMimeTypeIndexable(
+        transformationConnectionNames,transformationDescriptionStrings,
+        outputConnectionName,outputDescriptionString,mimeType,
+        checkActivity);
     }
 
     /** Check whether a document is indexable by the currently specified output connector.
     *@param localFile is the local copy of the file to check.
     *@return true if the document is indexable.
     */
+    @Override
     public boolean checkDocumentIndexable(File localFile)
       throws ManifoldCFException, ServiceInterruption
     {
-      return ingester.checkDocumentIndexable(job.getOutputConnectionName(),outputVersion,localFile);
+      return ingester.checkDocumentIndexable(
+        transformationConnectionNames,transformationDescriptionStrings,
+        outputConnectionName,outputDescriptionString,localFile,
+        checkActivity);
     }
 
     /** Check whether a document of a specified length is indexable by the currently specified output connector.
     *@param length is the length to check.
     *@return true if the document is indexable.
     */
+    @Override
     public boolean checkLengthIndexable(long length)
       throws ManifoldCFException, ServiceInterruption
     {
-      return ingester.checkLengthIndexable(job.getOutputConnectionName(),outputVersion,length);
+      return ingester.checkLengthIndexable(
+        transformationConnectionNames,transformationDescriptionStrings,
+        outputConnectionName,outputDescriptionString,length,
+        checkActivity);
     }
 
     /** Pre-determine whether a document's URL is indexable by this connector.  This method is used by participating repository connectors
@@ -1298,10 +1353,14 @@ public class WorkerThread extends Thread
     *@param url is the URL of the document.
     *@return true if the file is indexable.
     */
+    @Override
     public boolean checkURLIndexable(String url)
       throws ManifoldCFException, ServiceInterruption
     {
-      return ingester.checkURLIndexable(job.getOutputConnectionName(),outputVersion,url);
+      return ingester.checkURLIndexable(
+        transformationConnectionNames,transformationDescriptionStrings,
+        outputConnectionName,outputDescriptionString,url,
+        checkActivity);
     }
 
     /** Record time-stamped information about the activity of the connector.
@@ -1320,6 +1379,7 @@ public class WorkerThread extends Thread
     *       described in the resultCode field.  This field is not meant to be queried on.  May be null.
     *@param childIdentifiers is a set of child entity identifiers associated with this activity.  May be null.
     */
+    @Override
     public void recordActivity(Long startTime, String activityType, Long dataSize,
       String entityIdentifier, String resultCode, String resultDescription, String[] childIdentifiers)
       throws ManifoldCFException
@@ -1333,10 +1393,11 @@ public class WorkerThread extends Thread
     *@param dataName is the name of the data items to retrieve.
     *@return an array containing the unique data values passed from ALL parents.  Note that these are in no particular order, and there will not be any duplicates.
     */
+    @Override
     public String[] retrieveParentData(String localIdentifier, String dataName)
       throws ManifoldCFException
     {
-      return jobManager.retrieveParentData(job.getID(),ManifoldCF.hash(localIdentifier),dataName);
+      return jobManager.retrieveParentData(jobID,ManifoldCF.hash(localIdentifier),dataName);
     }
 
     /** Retrieve data passed from parents to a specified child document.
@@ -1344,10 +1405,11 @@ public class WorkerThread extends Thread
     *@param dataName is the name of the data items to retrieve.
     *@return an array containing the unique data values passed from ALL parents.  Note that these are in no particular order, and there will not be any duplicates.
     */
+    @Override
     public CharacterInput[] retrieveParentDataAsFiles(String localIdentifier, String dataName)
       throws ManifoldCFException
     {
-      return jobManager.retrieveParentDataAsFiles(job.getID(),ManifoldCF.hash(localIdentifier),dataName);
+      return jobManager.retrieveParentDataAsFiles(jobID,ManifoldCF.hash(localIdentifier),dataName);
     }
 
     /** Check whether current job is still active.
@@ -1355,10 +1417,11 @@ public class WorkerThread extends Thread
     * itself being aborted.  If the connector should abort, this method will raise a properly-formed ServiceInterruption, which if thrown to the
     * caller, will signal that the current versioning activity remains incomplete and must be retried when the job is resumed.
     */
+    @Override
     public void checkJobStillActive()
       throws ManifoldCFException, ServiceInterruption
     {
-      if (jobManager.checkJobActive(job.getID()) == false)
+      if (jobManager.checkJobActive(jobID) == false)
         throw new ServiceInterruption("Job no longer active",System.currentTimeMillis(),true);
     }
 
@@ -1369,6 +1432,7 @@ public class WorkerThread extends Thread
     *@param eventName is the event name.
     *@return false if the event is already in the "pending" state.
     */
+    @Override
     public boolean beginEventSequence(String eventName)
       throws ManifoldCFException
     {
@@ -1382,6 +1446,7 @@ public class WorkerThread extends Thread
     * the sole right to complete it.  Otherwise, race conditions can develop which would be difficult to diagnose.
     *@param eventName is the event name.
     */
+    @Override
     public void completeEventSequence(String eventName)
       throws ManifoldCFException
     {
@@ -1394,17 +1459,19 @@ public class WorkerThread extends Thread
     * presumed that the reason for the requeue is because of sequencing issues synchronized around an underlying event.
     *@param localIdentifier is the document identifier to requeue
     */
+    @Override
     public void retryDocumentProcessing(String localIdentifier)
       throws ManifoldCFException
     {
       // Accumulate aborts
-      abortSet.put(localIdentifier,localIdentifier);
+      abortSet.add(localIdentifier);
     }
 
     /** Create a global string from a simple string.
     *@param simpleString is the simple string.
     *@return a global string.
     */
+    @Override
     public String createGlobalString(String simpleString)
     {
       return ManifoldCF.createGlobalString(simpleString);
@@ -1414,6 +1481,7 @@ public class WorkerThread extends Thread
     *@param simpleString is the simple string.
     *@return a connection-specific string.
     */
+    @Override
     public String createConnectionSpecificString(String simpleString)
     {
       return ManifoldCF.createConnectionSpecificString(connectionName,simpleString);
@@ -1423,9 +1491,10 @@ public class WorkerThread extends Thread
     *@param simpleString is the simple string.
     *@return a job-specific string.
     */
+    @Override
     public String createJobSpecificString(String simpleString)
     {
-      return ManifoldCF.createJobSpecificString(job.getID(),simpleString);
+      return ManifoldCF.createJobSpecificString(jobID,simpleString);
     }
 
   }
@@ -1435,20 +1504,31 @@ public class WorkerThread extends Thread
   protected static class ProcessActivity implements IProcessActivity
   {
     // Member variables
+    protected final Long jobID;
     protected final String processID;
     protected final IThreadContext threadContext;
     protected final IJobManager jobManager;
     protected final IIncrementalIngester ingester;
+    protected final String connectionName;
+    protected final String outputName;
+    protected final String[] transformationConnectionNames;
+    protected final String outputDescriptionString;
+    protected final String[] transformationDescriptionStrings;
     protected final long currentTime;
-    protected final IJobDescription job;
+    protected final Long expireInterval;
+    protected final Map<String,Set<String>> forcedMetadata;
+    protected final Long recrawlInterval;
+    protected final Long maxInterval;
+    protected final int hopcountMode;
     protected final IRepositoryConnection connection;
     protected final IRepositoryConnector connector;
     protected final IRepositoryConnectionManager connMgr;
     protected final String[] legalLinkTypes;
     protected final OutputActivity ingestLogger;
     protected final IReprioritizationTracker rt;
-    protected final HashMap abortSet;
+    protected final Set<String> abortSet;
     protected final String outputVersion;
+    protected final String transformationVersion;
     protected final String parameterVersion;
     
     // We submit references in bulk, because that's way more efficient.
@@ -1467,20 +1547,40 @@ public class WorkerThread extends Thread
     *@param jobManager is the job manager
     *@param ingester is the ingester
     */
-    public ProcessActivity(String processID, IThreadContext threadContext,
+    public ProcessActivity(Long jobID, String processID,
+      IThreadContext threadContext,
       IReprioritizationTracker rt, IJobManager jobManager,
-      IIncrementalIngester ingester, long currentTime,
-      IJobDescription job, IRepositoryConnection connection, IRepositoryConnector connector,
+      IIncrementalIngester ingester,
+      String connectionName, String outputName, String[] transformationConnectionNames,
+      String outputDescriptionString, String[] transformationDescriptionStrings,
+      long currentTime,
+      Long expireInterval,
+      Map<String,Set<String>> forcedMetadata,
+      Long recrawlInterval,
+      Long maxInterval,
+      int hopcountMode,
+      IRepositoryConnection connection, IRepositoryConnector connector,
       IRepositoryConnectionManager connMgr, String[] legalLinkTypes, OutputActivity ingestLogger,
-      HashMap abortSet, String outputVersion, String parameterVersion)
+      Set<String> abortSet,
+      String outputVersion, String transformationVersion, String parameterVersion)
     {
+      this.jobID = jobID;
       this.processID = processID;
       this.threadContext = threadContext;
       this.rt = rt;
       this.jobManager = jobManager;
       this.ingester = ingester;
+      this.connectionName = connectionName;
+      this.outputName = outputName;
+      this.outputDescriptionString = outputDescriptionString;
+      this.transformationConnectionNames = transformationConnectionNames;
+      this.transformationDescriptionStrings = transformationDescriptionStrings;
       this.currentTime = currentTime;
-      this.job = job;
+      this.expireInterval = expireInterval;
+      this.forcedMetadata = forcedMetadata;
+      this.recrawlInterval = recrawlInterval;
+      this.maxInterval = maxInterval;
+      this.hopcountMode = hopcountMode;
       this.connection = connection;
       this.connector = connector;
       this.connMgr = connMgr;
@@ -1489,6 +1589,7 @@ public class WorkerThread extends Thread
       this.abortSet = abortSet;
       this.outputVersion = outputVersion;
       this.parameterVersion = parameterVersion;
+      this.transformationVersion = transformationVersion;
     }
 
     /** Clean up any dangling information, before abandoning this process activity object */
@@ -1531,7 +1632,7 @@ public class WorkerThread extends Thread
       +" to '"+localIdentifier+"', relationship type "+((relationshipType==null)?"null":"'"+relationshipType+"'")
       +", with "+((dataNames==null)?"no":Integer.toString(dataNames.length))+" data types, origination time="+((originationTime==null)?"unknown":originationTime.toString()));
 
-      Long expireInterval = job.getExpiration();
+      //Long expireInterval = job.getExpiration();
       if (expireInterval != null)
       {
         // We should not queue documents that have already expired; it wastes time
@@ -1679,7 +1780,7 @@ public class WorkerThread extends Thread
     public String[] retrieveParentData(String localIdentifier, String dataName)
       throws ManifoldCFException
     {
-      return jobManager.retrieveParentData(job.getID(),ManifoldCF.hash(localIdentifier),dataName);
+      return jobManager.retrieveParentData(jobID,ManifoldCF.hash(localIdentifier),dataName);
     }
 
     /** Retrieve data passed from parents to a specified child document.
@@ -1691,7 +1792,7 @@ public class WorkerThread extends Thread
     public CharacterInput[] retrieveParentDataAsFiles(String localIdentifier, String dataName)
       throws ManifoldCFException
     {
-      return jobManager.retrieveParentDataAsFiles(job.getID(),ManifoldCF.hash(localIdentifier),dataName);
+      return jobManager.retrieveParentDataAsFiles(jobID,ManifoldCF.hash(localIdentifier),dataName);
     }
 
     /** Record a document version, but don't ingest it.
@@ -1704,7 +1805,7 @@ public class WorkerThread extends Thread
       throws ManifoldCFException, ServiceInterruption
     {
       String documentIdentifierHash = ManifoldCF.hash(documentIdentifier);
-      ingester.documentRecord(job.getOutputConnectionName(),job.getConnectionName(),documentIdentifierHash,version,currentTime,ingestLogger);
+      ingester.documentRecord(outputName,connectionName,documentIdentifierHash,version,currentTime,ingestLogger);
     }
 
     /** Ingest the current document.
@@ -1727,7 +1828,7 @@ public class WorkerThread extends Thread
 
       if (data != null)
       {
-        Map<String,Set<String>> forcedMetadata = job.getForcedMetadata();
+        //Map<String,Set<String>> forcedMetadata = job.getForcedMetadata();
         
         // Modify the repository document with forced parameters.
         for (String paramName : forcedMetadata.keySet())
@@ -1744,9 +1845,12 @@ public class WorkerThread extends Thread
       }
         
       // First, we need to add into the metadata the stuff from the job description.
-      ingester.documentIngest(job.getOutputConnectionName(),
-        job.getConnectionName(),documentIdentifierHash,
-        version,outputVersion,parameterVersion,
+      ingester.documentIngest(transformationConnectionNames,
+        transformationDescriptionStrings,
+        outputName,
+        outputDescriptionString,
+        connectionName,documentIdentifierHash,
+        version,transformationVersion,outputVersion,parameterVersion,
         connection.getACLAuthority(),
         data,currentTime,
         documentURI,
@@ -1780,8 +1884,8 @@ public class WorkerThread extends Thread
       throws ManifoldCFException, ServiceInterruption
     {
       String documentIdentifierHash = ManifoldCF.hash(documentIdentifier);
-      ingester.documentDelete(job.getOutputConnectionName(),
-        job.getConnectionName(),documentIdentifierHash,
+      ingester.documentDelete(outputName,
+        connectionName,documentIdentifierHash,
         ingestLogger);
     }
 
@@ -1868,10 +1972,10 @@ public class WorkerThread extends Thread
     public Long calculateDocumentRescheduleTime(long currentTime, long timeAmt, String localIdentifier)
     {
       Long recrawlTime = null;
-      Long recrawlInterval = job.getInterval();
+      //Long recrawlInterval = job.getInterval();
       if (recrawlInterval != null)
       {
-        Long maxInterval = job.getMaxInterval();
+        //Long maxInterval = job.getMaxInterval();
         long actualInterval = recrawlInterval.longValue() + timeAmt;
         if (maxInterval != null && actualInterval > maxInterval.longValue())
           actualInterval = maxInterval.longValue();
@@ -1908,7 +2012,7 @@ public class WorkerThread extends Thread
       Long originationTime = getDocumentOriginationTime(localIdentifier);
       if (originationTime == null)
         originationTime = new Long(currentTime);
-      Long expireInterval = job.getExpiration();
+      //Long expireInterval = job.getExpiration();
       Long expireTime = null;
       if (expireInterval != null)
         expireTime = new Long(originationTime.longValue() + expireInterval.longValue());
@@ -2024,7 +2128,7 @@ public class WorkerThread extends Thread
         rt.preloadBinValues();
 
         jobManager.addDocuments(processID,
-          job.getID(),legalLinkTypes,docidHashes,docids,db.getParentIdentifierHash(),db.getLinkType(),job.getHopcountMode(),
+          jobID,legalLinkTypes,docidHashes,docids,db.getParentIdentifierHash(),db.getLinkType(),hopcountMode,
           dataNames,dataValues,currentTime,priorities,eventNames);
         
         rt.clearPreloadedValues();
@@ -2042,7 +2146,7 @@ public class WorkerThread extends Thread
     public void checkJobStillActive()
       throws ManifoldCFException, ServiceInterruption
     {
-      if (jobManager.checkJobActive(job.getID()) == false)
+      if (jobManager.checkJobActive(jobID) == false)
         throw new ServiceInterruption("Job no longer active",System.currentTimeMillis());
     }
 
@@ -2085,7 +2189,7 @@ public class WorkerThread extends Thread
       throws ManifoldCFException
     {
       // Accumulate aborts
-      abortSet.put(localIdentifier,localIdentifier);
+      abortSet.add(localIdentifier);
     }
 
     /** Check whether a mime type is indexable by the currently specified output connector.
@@ -2096,7 +2200,10 @@ public class WorkerThread extends Thread
     public boolean checkMimeTypeIndexable(String mimeType)
       throws ManifoldCFException, ServiceInterruption
     {
-      return ingester.checkMimeTypeIndexable(job.getOutputConnectionName(),outputVersion,mimeType);
+      return ingester.checkMimeTypeIndexable(
+        transformationConnectionNames,transformationDescriptionStrings,
+        outputName,outputDescriptionString,mimeType,
+        ingestLogger);
     }
 
     /** Check whether a document is indexable by the currently specified output connector.
@@ -2107,7 +2214,10 @@ public class WorkerThread extends Thread
     public boolean checkDocumentIndexable(File localFile)
       throws ManifoldCFException, ServiceInterruption
     {
-      return ingester.checkDocumentIndexable(job.getOutputConnectionName(),outputVersion,localFile);
+      return ingester.checkDocumentIndexable(
+        transformationConnectionNames,transformationDescriptionStrings,
+        outputName,outputDescriptionString,localFile,
+        ingestLogger);
     }
 
     /** Check whether a document of a specified length is indexable by the currently specified output connector.
@@ -2118,7 +2228,10 @@ public class WorkerThread extends Thread
     public boolean checkLengthIndexable(long length)
       throws ManifoldCFException, ServiceInterruption
     {
-      return ingester.checkLengthIndexable(job.getOutputConnectionName(),outputVersion,length);
+      return ingester.checkLengthIndexable(
+        transformationConnectionNames,transformationDescriptionStrings,
+        outputName,outputDescriptionString,length,
+        ingestLogger);
     }
 
     /** Pre-determine whether a document's URL is indexable by this connector.  This method is used by participating repository connectors
@@ -2130,7 +2243,10 @@ public class WorkerThread extends Thread
     public boolean checkURLIndexable(String url)
       throws ManifoldCFException, ServiceInterruption
     {
-      return ingester.checkURLIndexable(job.getOutputConnectionName(),outputVersion,url);
+      return ingester.checkURLIndexable(
+        transformationConnectionNames,transformationDescriptionStrings,
+        outputName,outputDescriptionString,url,
+        ingestLogger);
     }
 
     /** Create a global string from a simple string.
@@ -2160,7 +2276,7 @@ public class WorkerThread extends Thread
     @Override
     public String createJobSpecificString(String simpleString)
     {
-      return ManifoldCF.createJobSpecificString(job.getID(),simpleString);
+      return ManifoldCF.createJobSpecificString(jobID,simpleString);
     }
 
   }
@@ -2428,23 +2544,78 @@ public class WorkerThread extends Thread
     }
   }
 
+  /** The check activity class */
+  protected static class CheckActivity implements IOutputCheckActivity
+  {
+    public CheckActivity()
+    {
+    }
+
+    /** Detect if a mime type is acceptable downstream or not.  This method is used to determine whether it makes sense to fetch a document
+    * in the first place.
+    *@param mimeType is the mime type of the document.
+    *@return true if the mime type can be accepted by the downstream connection.
+    */
+    @Override
+    public boolean checkMimeTypeIndexable(String mimeType)
+      throws ManifoldCFException, ServiceInterruption
+    {
+      return false;
+    }
+
+    /** Pre-determine whether a document (passed here as a File object) is acceptable downstream.  This method is
+    * used to determine whether a document needs to be actually transferred.  This hook is provided mainly to support
+    * search engines that only handle a small set of accepted file types.
+    *@param localFile is the local file to check.
+    *@return true if the file is acceptable by the downstream connection.
+    */
+    @Override
+    public boolean checkDocumentIndexable(File localFile)
+      throws ManifoldCFException, ServiceInterruption
+    {
+      return false;
+    }
+
+    /** Pre-determine whether a document's length is acceptable downstream.  This method is used
+    * to determine whether to fetch a document in the first place.
+    *@param length is the length of the document.
+    *@return true if the file is acceptable by the downstream connection.
+    */
+    @Override
+    public boolean checkLengthIndexable(long length)
+      throws ManifoldCFException, ServiceInterruption
+    {
+      return false;
+    }
+
+    /** Pre-determine whether a document's URL is acceptable downstream.  This method is used
+    * to help filter out documents that cannot be indexed in advance.
+    *@param url is the URL of the document.
+    *@return true if the file is acceptable by the downstream connection.
+    */
+    @Override
+    public boolean checkURLIndexable(String url)
+      throws ManifoldCFException, ServiceInterruption
+    {
+      return false;
+    }
+    
+  }
+  
   /** The ingest logger class */
-  protected static class OutputActivity implements IOutputActivity
+  protected static class OutputActivity extends CheckActivity implements IOutputActivity
   {
 
     // Connection name
-    protected String connectionName;
+    protected final String connectionName;
     // Connection manager
-    protected IRepositoryConnectionManager connMgr;
-    // Output connection name
-    protected String outputConnectionName;
+    protected final IRepositoryConnectionManager connMgr;
 
     /** Constructor */
-    public OutputActivity(String connectionName, IRepositoryConnectionManager connMgr, String outputConnectionName)
+    public OutputActivity(String connectionName, IRepositoryConnectionManager connMgr)
     {
       this.connectionName = connectionName;
       this.connMgr = connMgr;
-      this.outputConnectionName = outputConnectionName;
     }
 
     /** Record time-stamped information about the activity of the output connector.
@@ -2462,11 +2633,12 @@ public class WorkerThread extends Thread
     *@param resultDescription is a (possibly long) human-readable string which adds detail, if required, to the result
     *       described in the resultCode field.  This field is not meant to be queried on.  May be null.
     */
+    @Override
     public void recordActivity(Long startTime, String activityType, Long dataSize,
       String entityURI, String resultCode, String resultDescription)
       throws ManifoldCFException
     {
-      connMgr.recordHistory(connectionName,startTime,ManifoldCF.qualifyOutputActivityName(activityType,outputConnectionName),dataSize,entityURI,resultCode,
+      connMgr.recordHistory(connectionName,startTime,activityType,dataSize,entityURI,resultCode,
         resultDescription,null);
     }
 
@@ -2476,6 +2648,7 @@ public class WorkerThread extends Thread
     *@param accessToken is the raw, repository access token.
     *@return the properly qualified access token.
     */
+    @Override
     public String qualifyAccessToken(String authorityNameString, String accessToken)
       throws ManifoldCFException
     {
@@ -2485,6 +2658,19 @@ public class WorkerThread extends Thread
           return URLEncoder.encode(authorityNameString) + ":" + URLEncoder.encode(accessToken);
     }
 
+    /** Send a document via the pipeline to the next output connection.
+    *@param documentURI is the document's URI.
+    *@param document is the document data to be processed (handed to the output data store).
+    *@param authorityNameString is the authority name string that should be used to qualify the document's access tokens.
+    *@return the document status (accepted or permanently rejected); return codes are listed in IPipelineConnector.
+    */
+    public int sendDocument(String documentURI, RepositoryDocument document, String authorityNameString)
+      throws ManifoldCFException, ServiceInterruption
+    {
+      // No downstream connection at output connection level.
+      return IPipelineConnector.DOCUMENTSTATUS_REJECTED;
+    }
+
   }
 
 }

Modified: manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties?rev=1601529&r1=1601528&r2=1601529&view=diff
==============================================================================
--- manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties (original)
+++ manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties Mon Jun  9 23:19:08 2014
@@ -13,9 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-editoutput.Name=Name
-editoutput.Type=Type
-editoutput.Throttling=Throttling
 editconnection.Name=Name
 editconnection.Type=Type
 editconnection.Throttling=Throttling
@@ -48,6 +45,7 @@ banner.DocumentIngestion=Document Ingest
 
 navigation.LogOut=Log Out
 navigation.Outputs=Outputs
+navigation.ListTransformationConnections=List Transformation Connections
 navigation.ListOutputConnections=List Output Connections
 navigation.Authorities=Authorities
 navigation.ListUserMappings=List User Mapping Connections
@@ -69,6 +67,7 @@ navigation.ResultHistogram=Result Histog
 navigation.Miscellaneous=Miscellaneous
 navigation.Locale=en_US
 navigation.Help=Help
+navigation.Listtransformationconnections=List transformation connections
 navigation.Listoutputconnections=List output connections
 navigation.Listrepositoryconnections=List repository connections
 navigation.Listusermappings=List user mapping connections
@@ -98,6 +97,9 @@ listoutputs.AddAnOutputConnection=Add an
 listoutputs.uninstalled=(uninstalled)
 
 editoutput.ApacheManifoldCFEditOutputConnection=Apache ManifoldCF: Edit Output Connection
+editoutput.Name=Name
+editoutput.Type=Type
+editoutput.Throttling=Throttling
 editoutput.EditAnOutputConnection=Edit an Output Connection
 editoutput.EditOutputConnection2=Edit Output Connection
 editoutput.ConnectionTypeColon=Connection type:
@@ -113,7 +115,6 @@ editoutput.NameColon=Name:
 editoutput.DescriptionColon=Description:
 editoutput.ConnectionMustHaveAName=Connection must have a name
 editoutput.TheMaximumNumberOfConnectionsMustBeAValidInteger=The maximum number of connections must be a valid integer
-
 editoutput.UNREGISTERED=UNREGISTERED
 editoutput.tab=tab
 
@@ -143,6 +144,60 @@ viewoutput.uninstalled=(uninstalled)
 viewoutput.Connectorisnotinstalled=Connector is not installed.
 viewoutput.Threwexception=Threw exception:
 
+listtransformations.ApacheManifoldCFListTransformationConnections=Apache ManifoldCF: List Transformation Connections
+listtransformations.ListOfTransformationConnections=List of Transformation Connections
+listtransformations.Name=Name
+listtransformations.Description=Description
+listtransformations.ConnectionType=Connection Type
+listtransformations.Max=Max
+listtransformations.AddaNewTransformationConnection=Add a new transformation connection
+listtransformations.View=View
+listtransformations.Edit=Edit
+listtransformations.Delete=Delete
+listtransformations.DeleteTransformationConnection=Delete transformation connection
+listtransformations.AddATransformationConnection=Add a transformation connection
+listtransformations.uninstalled=(uninstalled)
+
+edittransformation.ApacheManifoldCFEditTransformationConnection=Apache ManifoldCF: Edit Transformation Connection
+edittransformation.Name=Name
+edittransformation.Type=Type
+edittransformation.Throttling=Throttling
+edittransformation.EditATransformationConnection=Edit a Transformation Connection
+edittransformation.EditTransformationConnection2=Edit Transformation Connection
+edittransformation.ConnectionTypeColon=Connection type:
+edittransformation.Continue=Continue
+edittransformation.ContinueToNextPage=Continue to next page
+edittransformation.Cancel=Cancel
+edittransformation.CancelTransformationConnectionEditing=Cancel transformation connection editing
+edittransformation.Save=Save
+edittransformation.SaveThisTransformationConnection=Save this transformation connection
+edittransformation.MaxConnectionsColon=Max connections:
+edittransformation.EditTransformationConnection=Edit transformation connection
+edittransformation.NameColon=Name:
+edittransformation.DescriptionColon=Description:
+edittransformation.ConnectionMustHaveAName=Connection must have a name
+edittransformation.TheMaximumNumberOfConnectionsMustBeAValidInteger=The maximum number of connections must be a valid integer
+edittransformation.UNREGISTERED=UNREGISTERED
+edittransformation.tab=tab
+
+viewtransformation.ApacheManifoldCFViewTransformationConnectionStatus=Apache ManifoldCF: View Transformation Connection Status
+viewtransformation.ViewTransformationConnectionStatus=View Transformation Connection Status
+viewtransformation.NameColon=Name:
+viewtransformation.DescriptionColon=Description:
+viewtransformation.ConnectionTypeColon=Connection type:
+viewtransformation.MaxConnectionsColon=Max connections:
+viewtransformation.ConnectionStatusColon=Connection status:
+viewtransformation.Refresh=Refresh
+viewtransformation.EditThisTransformationConnection=Edit this transformation connection
+viewtransformation.Edit=Edit
+viewtransformation.DeleteThisTransformationConnection=Delete this transformation connection
+viewtransformation.Delete=Delete
+viewtransformation.Deletetransformationconnection=Delete transformation connection
+viewtransformation.qmark=?
+viewtransformation.uninstalled=(uninstalled)
+viewtransformation.Connectorisnotinstalled=Connector is not installed.
+viewtransformation.Threwexception=Threw exception:
+
 listgroups.ApacheManifoldCFListAuthorityGroups=Apache ManifoldCF: List Authority Groups
 listgroups.DeleteAuthorityGroup=Delete authority group
 listgroups.ListOfAuthorityGroups=List of Authority Groups
@@ -345,6 +400,18 @@ listjobs.CopyJob=Copy job
 editjob.ApacheManifoldCFEditJob=Apache ManifoldCF: Edit Job
 editjob.EditaJob=Edit a Job
 editjob.EditJob=Edit job
+
+editjob.PipelineColon=Pipeline:
+editjob.StageNumber=Stage
+editjob.StageDescription=Description
+editjob.StageConnectionName=Connection name
+editjob.RepositoryStage=(Repository)
+editjob.OutputStage=(Output)
+editjob.InsertBefore=Insert before
+editjob.Insertnewstagehere=Insert new stage here
+editjob.SelectAPipelineStageConnectionName=Select a pipeline stage connection name
+editjob.Deletestage=Delete stage
+
 editjob.NoRepositoryConnectionsDefinedCreateOneFirst=No repository connections defined - create one first
 editjob.NoOutputConnectionsDefinedCreateOneFirst=No output connections defined - create one first
 editjob.NameColon=Name:
@@ -774,6 +841,14 @@ error.Return=Return
 
 viewjob.ApacheManifoldCFViewJob=Apache ManifoldCF: View Job
 viewjob.ViewAJob=View a Job
+
+viewjob.PipelineColon=Pipeline:
+viewjob.StageNumber=Stage
+viewjob.StageDescription=Description
+viewjob.StageConnectionName=Connection name
+viewjob.RepositoryStage=(Repository)
+viewjob.OutputStage=(Output)
+
 viewjob.ScheduleTypeColon=Schedule type:
 viewjob.MinimumRecrawlIntervalColon=Minimum recrawl interval:
 viewjob.MaximumRecrawlIntervalColon=Maximum recrawl interval: