You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/07/18 20:37:49 UTC
svn commit: r1611760 - in /manifoldcf/branches/CONNECTORS-989/framework:
agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/
agents/src/main/java/org/apache/manifoldcf/agents/interfaces/
pull-agent/src/main/java/org/apache/manifoldcf/c...
Author: kwright
Date: Fri Jul 18 18:37:49 2014
New Revision: 1611760
URL: http://svn.apache.org/r1611760
Log:
Begin process of getting component IDs into connector, where decisions will be made
Modified:
manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IngestStatuses.java
manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
Modified: manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1611760&r1=1611759&r2=1611760&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java (original)
+++ manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java Fri Jul 18 18:37:49 2014
@@ -1511,7 +1511,8 @@ public class IncrementalIngester extends
if (authorityName == null)
authorityName = "";
int indexValue = position.intValue();
- rval.addStatus(identifierClasses[indexValue],identifierHashes[indexValue],outputConnectionName,
+ // MHL
+ rval.addStatus(identifierClasses[indexValue],identifierHashes[indexValue],null,outputConnectionName,
new DocumentIngestStatus(lastVersion,lastTransformationVersion,lastOutputVersion,paramVersion,authorityName));
}
}
Modified: manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IngestStatuses.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IngestStatuses.java?rev=1611760&r1=1611759&r2=1611760&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IngestStatuses.java (original)
+++ manifoldcf/branches/CONNECTORS-989/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IngestStatuses.java Fri Jul 18 18:37:49 2014
@@ -28,7 +28,7 @@ public class IngestStatuses
{
public static final String _rcsid = "@(#)$Id$";
- protected final Map<OutputKey,DocumentIngestStatus> statuses = new HashMap<OutputKey,DocumentIngestStatus>();
+ protected final Map<OutputKey,Map<String,DocumentIngestStatus>> statuses = new HashMap<OutputKey,Map<String,DocumentIngestStatus>>();
public IngestStatuses()
{
@@ -37,23 +37,50 @@ public class IngestStatuses
/** Add a status record.
*@param documentClass is the document class.
*@param documentIDHash is the document id's hash value.
+ *@param componentIDHash is the component id hash value, if any.
*@param outputConnectionName is the output connection name.
*@param status is the status record.
*/
- public void addStatus(String documentClass, String documentIDHash, String outputConnectionName, DocumentIngestStatus status)
+ public void addStatus(String documentClass, String documentIDHash, String componentIDHash, String outputConnectionName, DocumentIngestStatus status)
{
- statuses.put(new OutputKey(documentClass,documentIDHash,outputConnectionName),status);
+ if (componentIDHash == null)
+ componentIDHash = "";
+ OutputKey ok = new OutputKey(documentClass,documentIDHash,outputConnectionName);
+ Map<String,DocumentIngestStatus> map = statuses.get(ok);
+ if (map == null)
+ {
+ map = new HashMap<String,DocumentIngestStatus>();
+ statuses.put(ok,map);
+ }
+ map.put(componentIDHash,status);
+ }
+
+ /** Get the set of component hashes for a given output.
+ *@param documentClass is the document class.
+ *@param documentIDHash is the document id's hash value.
+ *@param outputConnectionName is the output connection name.
+ *@return the set of component hashes (empty string meaning no component).
+ */
+ public Set<String> componentIterator(String documentClass, String documentIDHash, String outputConnectionName)
+ {
+ Map<String,DocumentIngestStatus> map = statuses.get(new OutputKey(documentClass,documentIDHash,outputConnectionName));
+ if (map == null)
+ return new HashSet<String>();
+ return map.keySet();
}
/** Retrieve a status record.
*@param documentClass is the document class.
*@param documentIDHash is the document id's hash value.
+ *@param componentIDHash is the component id hash value, if any.
*@param outputConnectionName is the output connection name.
*@return the status record, if record.
*/
- public DocumentIngestStatus getStatus(String documentClass, String documentIDHash, String outputConnectionName)
+ public DocumentIngestStatus getStatus(String documentClass, String documentIDHash, String componentIDHash, String outputConnectionName)
{
- return statuses.get(new OutputKey(documentClass,documentIDHash,outputConnectionName));
+ if (componentIDHash == null)
+ componentIDHash = "";
+ return statuses.get(new OutputKey(documentClass,documentIDHash,outputConnectionName)).get(componentIDHash);
}
protected static class OutputKey
Modified: manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java?rev=1611760&r1=1611759&r2=1611760&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java (original)
+++ manifoldcf/branches/CONNECTORS-989/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java Fri Jul 18 18:37:49 2014
@@ -257,7 +257,8 @@ public class StufferThread extends Threa
for (int j = 0; j < pipelineSpecifications[i].getOutputCount(); j++)
{
String outputName = pipelineSpecifications[i].getStageConnectionName(pipelineSpecifications[i].getOutputStage(j));
- DocumentIngestStatus status = statuses.getStatus(documentClasses[i],documentIDHashes[i],outputName);
+ // MHL
+ DocumentIngestStatus status = statuses.getStatus(documentClasses[i],documentIDHashes[i],null,outputName);
if (status != null)
versions[i].put(outputName,status);
}