You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/01/17 14:02:36 UTC
svn commit: r1434653 - in /manifoldcf/trunk:
connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/
connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/
connectors/webcrawler/connector/src/main...
Author: kwright
Date: Thu Jan 17 13:02:36 2013
New Revision: 1434653
URL: http://svn.apache.org/viewvc?rev=1434653&view=rev
Log:
Provide a mechanism to communicate a mime type (if known) from repository connector to output connector, and hook this up into the Solr, Web, and RSS connectors. Part of CONNECTORS-613.
Modified:
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
Modified: manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java?rev=1434653&r1=1434652&r2=1434653&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java (original)
+++ manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/RSSConnector.java Thu Jan 17 13:02:36 2013
@@ -1355,6 +1355,9 @@ public class RSSConnector extends org.ap
long dataSize = cache.getDataLength(urlValue);
RepositoryDocument rd = new RepositoryDocument();
+ // Set content type
+ rd.setMimeType(cache.getContentType(urlValue));
+
// Turn into acls and add into description
String[] aclArray = new String[acls.size()];
int j = 0;
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1434653&r1=1434652&r2=1434653&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java Thu Jan 17 13:02:36 2013
@@ -736,6 +736,7 @@ public class HttpPoster
{
long length = document.getBinaryLength();
InputStream is = document.getBinaryStream();
+ String contentType = document.getMimeType();
try
{
@@ -786,7 +787,7 @@ public class HttpPoster
contentStreamUpdateRequest.setParams(out);
- contentStreamUpdateRequest.addContentStream(new RepositoryDocumentStream(is,length));
+ contentStreamUpdateRequest.addContentStream(new RepositoryDocumentStream(is,length,contentType));
// Fire off the request.
// Note: I need to know whether the document has been permanently rejected or not, but we currently have
@@ -1089,13 +1090,15 @@ public class HttpPoster
*/
protected static class RepositoryDocumentStream extends ContentStreamBase
{
- protected InputStream is;
- protected long length;
+ protected final InputStream is;
+ protected final long length;
+ protected final String contentType;
- public RepositoryDocumentStream(InputStream is, long length)
+ public RepositoryDocumentStream(InputStream is, long length, String contentType)
{
this.is = is;
this.length = length;
+ this.contentType = contentType;
}
@Override
@@ -1119,7 +1122,7 @@ public class HttpPoster
@Override
public String getContentType()
{
- return "application/octet-stream";
+ return contentType;
}
}
Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java?rev=1434653&r1=1434652&r2=1434653&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java (original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java Thu Jan 17 13:02:36 2013
@@ -1349,6 +1349,9 @@ public class WebcrawlerConnector extends
RepositoryDocument rd = new RepositoryDocument();
+ // Set the content type
+ rd.setMimeType(cache.getContentType(documentIdentifier));
+
// Turn into acls and add into description
String[] aclArray = new String[acls.size()];
int j = 0;
Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java?rev=1434653&r1=1434652&r2=1434653&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java (original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java Thu Jan 17 13:02:36 2013
@@ -45,6 +45,7 @@ public class RepositoryDocument
protected Security shareSecurity = new Security();
protected List<Security> directorySecurity = new ArrayList<Security>();
protected String fileName = "docname";
+ protected String contentMimeType = "application/octet-stream";
/** Constructor.
*/
@@ -52,6 +53,22 @@ public class RepositoryDocument
{
}
+ /** Set the document's mime type.
+ *@param mimeType is the mime type.
+ */
+ public void setMimeType(String mimeType)
+ {
+ contentMimeType = mimeType;
+ }
+
+ /** Get the document's mime type.
+ *@return the mime type.
+ */
+ public String getMimeType()
+ {
+ return contentMimeType;
+ }
+
/** Set the document's "file" allow acls.
*@param acl is the allowed "file" access control token list for the document.
*/