You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/06/27 01:14:56 UTC
svn commit: r1605929 - in /manifoldcf/trunk: ./
connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/
connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/
Author: kwright
Date: Thu Jun 26 23:14:56 2014
New Revision: 1605929
URL: http://svn.apache.org/r1605929
Log:
Fix for CONNECTORS-981
Modified:
manifoldcf/trunk/ (props changed)
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties
Propchange: manifoldcf/trunk/
------------------------------------------------------------------------------
Merged /manifoldcf/branches/CONNECTORS-981:r1605049-1605773
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1605929&r1=1605928&r2=1605929&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Thu Jun 26 23:14:56 2014
@@ -3,6 +3,10 @@ $Id$
======================= 1.7-dev =====================
+CONNECTORS-981: Add support for SolrInputDocument indexing in
+Solr connector.
+(Alessandro Benedetti, Karl Wright)
+
CONNECTORS-979: Fix ant build so that documentum and filenet show
up.
(Karl Wright)
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1605929&r1=1605928&r2=1605929&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java Thu Jun 26 23:14:56 2014
@@ -36,6 +36,7 @@ import java.net.MalformedURLException;
import java.util.*;
import java.util.regex.*;
+import org.apache.http.Consts;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
import org.apache.http.auth.UsernamePasswordCredentials;
@@ -55,6 +56,7 @@ import org.apache.solr.common.params.Mod
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.SolrException;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.common.SolrInputDocument;
/**
@@ -82,25 +84,29 @@ public class HttpPoster
protected SolrServer solrServer = null;
// Action URI pieces
- private String postUpdateAction;
- private String postRemoveAction;
- private String postStatusAction;
+ private final String postUpdateAction;
+ private final String postRemoveAction;
+ private final String postStatusAction;
// Attribute names
- private String allowAttributeName;
- private String denyAttributeName;
- private String idAttributeName;
- private String modifiedDateAttributeName;
- private String createdDateAttributeName;
- private String indexedDateAttributeName;
- private String fileNameAttributeName;
- private String mimeTypeAttributeName;
+ private final String allowAttributeName;
+ private final String denyAttributeName;
+ private final String idAttributeName;
+ private final String modifiedDateAttributeName;
+ private final String createdDateAttributeName;
+ private final String indexedDateAttributeName;
+ private final String fileNameAttributeName;
+ private final String mimeTypeAttributeName;
+ private final String contentAttributeName;
+
+ // Whether we use extract/update handler or not
+ private final boolean useExtractUpdateHandler;
// Document max length
- private Long maxDocumentLength;
+ private final Long maxDocumentLength;
// Commit-within flag
- private String commitWithin;
+ private final String commitWithin;
// Constants we need
private static final String LITERAL = "literal.";
@@ -118,9 +124,9 @@ public class HttpPoster
String updatePath, String removePath, String statusPath,
String allowAttributeName, String denyAttributeName, String idAttributeName,
String modifiedDateAttributeName, String createdDateAttributeName, String indexedDateAttributeName,
- String fileNameAttributeName, String mimeTypeAttributeName,
+ String fileNameAttributeName, String mimeTypeAttributeName, String contentAttributeName,
Long maxDocumentLength,
- String commitWithin)
+ String commitWithin, boolean useExtractUpdateHandler)
throws ManifoldCFException
{
// These are the paths to the handlers in Solr that deal with the actions we need to do
@@ -138,6 +144,8 @@ public class HttpPoster
this.indexedDateAttributeName = indexedDateAttributeName;
this.fileNameAttributeName = fileNameAttributeName;
this.mimeTypeAttributeName = mimeTypeAttributeName;
+ this.contentAttributeName = contentAttributeName;
+ this.useExtractUpdateHandler = useExtractUpdateHandler;
this.maxDocumentLength = maxDocumentLength;
@@ -164,9 +172,9 @@ public class HttpPoster
String realm, String userID, String password,
String allowAttributeName, String denyAttributeName, String idAttributeName,
String modifiedDateAttributeName, String createdDateAttributeName, String indexedDateAttributeName,
- String fileNameAttributeName, String mimeTypeAttributeName,
+ String fileNameAttributeName, String mimeTypeAttributeName, String contentAttributeName,
IKeystoreManager keystoreManager, Long maxDocumentLength,
- String commitWithin)
+ String commitWithin, boolean useExtractUpdateHandler)
throws ManifoldCFException
{
// These are the paths to the handlers in Solr that deal with the actions we need to do
@@ -184,6 +192,8 @@ public class HttpPoster
this.indexedDateAttributeName = indexedDateAttributeName;
this.fileNameAttributeName = fileNameAttributeName;
this.mimeTypeAttributeName = mimeTypeAttributeName;
+ this.contentAttributeName = contentAttributeName;
+ this.useExtractUpdateHandler = useExtractUpdateHandler;
this.maxDocumentLength = maxDocumentLength;
@@ -506,12 +516,12 @@ public class HttpPoster
* @param document is the document structure to ingest.
* @param arguments are the configuration arguments to pass in the post. Key is argument name, value is a list of the argument values.
* @param keepAllMetadata
- *@param authorityNameString is the name of the governing authority for this document's acls, or null if none.
+ * @param authorityNameString is the name of the governing authority for this document's acls, or null if none.
* @param activities is the activities object, so we can report what's happening. @return true if the ingestion was successful, or false if the ingestion is illegal.
* @throws ManifoldCFException, ServiceInterruption
*/
public boolean indexPost(String documentURI,
- RepositoryDocument document, Map arguments, Map<String, List<String>> sourceTargets,
+ RepositoryDocument document, Map<String,List<String>> arguments, Map<String, List<String>> sourceTargets,
boolean keepAllMetadata, String authorityNameString, IOutputAddActivity activities)
throws ManifoldCFException, ServiceInterruption
{
@@ -544,7 +554,7 @@ public class HttpPoster
try
{
IngestThread t = new IngestThread(documentURI,document,arguments,keepAllMetadata,sourceTargets,
- aclsMap,denyAclsMap,commitWithin);
+ aclsMap,denyAclsMap);
try
{
t.start();
@@ -803,6 +813,18 @@ public class HttpPoster
}
}
+ /**
+ * Output an acl level in a SolrInputDocument
+ */
+ protected void writeACLsInSolrDoc( SolrInputDocument inputDoc, String aclType, String[] acl, String[] denyAcl )
+ {
+ String metadataACLName = allowAttributeName + aclType;
+ inputDoc.addField( metadataACLName, acl );
+
+ String metadataDenyACLName = denyAttributeName + aclType;
+ inputDoc.addField( metadataDenyACLName, denyAcl );
+ }
+
/** Killable thread that does ingestions.
* Java 1.5 stopped permitting thread interruptions to abort socket waits. As a result, it is impossible to get threads to shutdown cleanly that are doing
* such waits. So, the places where this happens are segregated in their own threads so that they can be just abandoned.
@@ -817,7 +839,6 @@ public class HttpPoster
protected final Map<String,List<String>> sourceTargets;
protected final Map<String,String[]> aclsMap;
protected final Map<String,String[]> denyAclsMap;
- protected final String commitWithin;
protected final boolean keepAllMetadata;
protected Long activityStart = null;
@@ -830,8 +851,7 @@ public class HttpPoster
public IngestThread(String documentURI, RepositoryDocument document,
Map<String, List<String>> arguments, boolean keepAllMetadata, Map<String, List<String>> sourceTargets,
- Map<String,String[]> aclsMap, Map<String,String[]> denyAclsMap,
- String commitWithin)
+ Map<String,String[]> aclsMap, Map<String,String[]> denyAclsMap)
{
super();
setDaemon(true);
@@ -841,7 +861,6 @@ public class HttpPoster
this.aclsMap = aclsMap;
this.denyAclsMap = denyAclsMap;
this.sourceTargets = sourceTargets;
- this.commitWithin = commitWithin;
this.keepAllMetadata=keepAllMetadata;
}
@@ -860,90 +879,34 @@ public class HttpPoster
// Open a socket to ingest, and to the response stream to get the post result
try
{
+ SolrInputDocument currentSolrDoc = new SolrInputDocument();
ContentStreamUpdateRequest contentStreamUpdateRequest = new ContentStreamUpdateRequest(postUpdateAction);
-
- ModifiableSolrParams out = new ModifiableSolrParams();
-
- // Write the id field
- writeField(out,LITERAL+idAttributeName,documentURI);
- // Write the rest of the attributes
- if (modifiedDateAttributeName != null)
- {
- Date date = document.getModifiedDate();
- if (date != null)
- // Write value
- writeField(out,LITERAL+modifiedDateAttributeName,DateParser.formatISO8601Date(date));
- }
- if (createdDateAttributeName != null)
- {
- Date date = document.getCreatedDate();
- if (date != null)
- // Write value
- writeField(out,LITERAL+createdDateAttributeName,DateParser.formatISO8601Date(date));
- }
- if (indexedDateAttributeName != null)
- {
- Date date = document.getIndexingDate();
- if (date != null)
- // Write value
- writeField(out,LITERAL+indexedDateAttributeName,DateParser.formatISO8601Date(date));
- }
- if (fileNameAttributeName != null)
- {
- String fileName = document.getFileName();
- if (fileName != null)
- writeField(out,LITERAL+fileNameAttributeName,fileName);
- }
- if (mimeTypeAttributeName != null)
+ if ( useExtractUpdateHandler )
{
- String mimeType = document.getMimeType();
- if (mimeType != null)
- writeField(out,LITERAL+mimeTypeAttributeName,mimeType);
+ buildExtractUpdateHandlerRequest( length, is, contentType, contentName,
+ contentStreamUpdateRequest );
}
-
- // Write the access token information
- // Both maps have the same keys.
- Iterator<String> typeIterator = aclsMap.keySet().iterator();
- while (typeIterator.hasNext())
- {
- String aclType = typeIterator.next();
- writeACLs(out,aclType,aclsMap.get(aclType),denyAclsMap.get(aclType));
- }
-
- // Write the arguments
- for (String name : arguments.keySet())
+ else
{
- List<String> values = arguments.get(name);
- writeField(out,name,values);
+ currentSolrDoc = buildSolrDocument( length, is );
}
- // Write the metadata, each in a field by itself
- buildSolrParamsFromMetadata(out);
-
- // These are unnecessary now in the case of non-solrcloud setups, because we overrode the SolrJ posting method to use multipart.
- //writeField(out,LITERAL+"stream_size",String.valueOf(length));
- //writeField(out,LITERAL+"stream_name",document.getFileName());
-
- // General hint for Tika
- if (document.getFileName() != null)
- writeField(out,"resource.name",document.getFileName());
-
- // Write the commitWithin parameter
- if (commitWithin != null)
- writeField(out,COMMITWITHIN_METADATA,commitWithin);
-
- contentStreamUpdateRequest.setParams(out);
-
- contentStreamUpdateRequest.addContentStream(new RepositoryDocumentStream(is,length,contentType,contentName));
-
// Fire off the request.
// Note: I need to know whether the document has been permanently rejected or not, but we currently have
// no means to determine that. Analysis of SolrServerExceptions that have been thrown is likely needed.
try
{
readFromDocumentStreamYet = true;
- UpdateResponse response = contentStreamUpdateRequest.process(solrServer);
-
+ UpdateResponse response;
+ if ( useExtractUpdateHandler )
+ {
+ response = contentStreamUpdateRequest.process( solrServer );
+ }
+ else
+ {
+ response = solrServer.add( currentSolrDoc );
+ }
+
// Successful completion
activityStart = new Long(fullStartTime);
activityBytes = new Long(length);
@@ -1008,6 +971,174 @@ public class HttpPoster
}
}
+ private SolrInputDocument buildSolrDocument( long length, InputStream is )
+ throws IOException
+ {
+ SolrInputDocument outputDoc = new SolrInputDocument();
+
+ // Write the id field
+ outputDoc.addField( idAttributeName, documentURI );
+
+ if (contentAttributeName != null)
+ {
+ // Copy the content into a string. This is a bad thing to do, but we have no choice given SolrJ architecture at this time.
+ // We enforce a size limit upstream.
+ Reader r = new InputStreamReader(is, Consts.UTF_8);
+ StringBuilder sb = new StringBuilder((int)length);
+ char[] buffer = new char[65536];
+ while (true)
+ {
+ int amt = r.read(buffer,0,buffer.length);
+ if (amt == -1)
+ break;
+ sb.append(buffer,0,amt);
+ }
+ outputDoc.addField( contentAttributeName, sb.toString() );
+ }
+
+ // Write the rest of the attributes
+ if ( modifiedDateAttributeName != null )
+ {
+ Date date = document.getModifiedDate();
+ if ( date != null )
+ {
+ outputDoc.addField( modifiedDateAttributeName, DateParser.formatISO8601Date( date ) );
+ }
+ }
+ if ( createdDateAttributeName != null )
+ {
+ Date date = document.getCreatedDate();
+ if ( date != null )
+ {
+ outputDoc.addField( createdDateAttributeName, DateParser.formatISO8601Date( date ) );
+ }
+
+ }
+ if ( indexedDateAttributeName != null )
+ {
+ Date date = document.getIndexingDate();
+ if ( date != null )
+ {
+ outputDoc.addField( indexedDateAttributeName, DateParser.formatISO8601Date( date ) );
+ }
+ }
+ if ( fileNameAttributeName != null )
+ {
+ String fileName = document.getFileName();
+ if ( fileName != null )
+ {
+ outputDoc.addField( fileNameAttributeName, fileName );
+ }
+ }
+ if ( mimeTypeAttributeName != null )
+ {
+ String mimeType = document.getMimeType();
+ if ( mimeType != null )
+ {
+ outputDoc.addField( mimeTypeAttributeName, mimeType );
+ }
+ }
+
+ Iterator<String> typeIterator = aclsMap.keySet().iterator();
+ while (typeIterator.hasNext())
+ {
+ String aclType = typeIterator.next();
+ writeACLsInSolrDoc(outputDoc,aclType,aclsMap.get(aclType),denyAclsMap.get(aclType));
+ }
+
+ // Write the arguments
+ for ( String name : arguments.keySet() )
+ {
+ List<String> values = arguments.get( name );
+ outputDoc.addField( name, values );
+ }
+
+ // Write the metadata, each in a field by itself
+ buildSolrParamsFromMetadata( outputDoc );
+
+ return outputDoc;
+ }
+
+ private void buildExtractUpdateHandlerRequest( long length, InputStream is, String contentType,
+ String contentName,
+ ContentStreamUpdateRequest contentStreamUpdateRequest )
+ throws IOException
+ {
+ ModifiableSolrParams out = new ModifiableSolrParams();
+
+ // Write the id field
+ writeField(out,LITERAL+idAttributeName,documentURI);
+ // Write the rest of the attributes
+ if (modifiedDateAttributeName != null)
+ {
+ Date date = document.getModifiedDate();
+ if (date != null)
+ // Write value
+ writeField(out,LITERAL+modifiedDateAttributeName,DateParser.formatISO8601Date(date));
+ }
+ if (createdDateAttributeName != null)
+ {
+ Date date = document.getCreatedDate();
+ if (date != null)
+ // Write value
+ writeField(out,LITERAL+createdDateAttributeName,DateParser.formatISO8601Date(date));
+ }
+ if (indexedDateAttributeName != null)
+ {
+ Date date = document.getIndexingDate();
+ if (date != null)
+ // Write value
+ writeField(out,LITERAL+indexedDateAttributeName,DateParser.formatISO8601Date(date));
+ }
+ if (fileNameAttributeName != null)
+ {
+ String fileName = document.getFileName();
+ if (fileName != null)
+ writeField(out,LITERAL+fileNameAttributeName,fileName);
+ }
+ if (mimeTypeAttributeName != null)
+ {
+ String mimeType = document.getMimeType();
+ if (mimeType != null)
+ writeField(out,LITERAL+mimeTypeAttributeName,mimeType);
+ }
+
+ // Write the access token information
+ // Both maps have the same keys.
+ Iterator<String> typeIterator = aclsMap.keySet().iterator();
+ while (typeIterator.hasNext())
+ {
+ String aclType = typeIterator.next();
+ writeACLs(out,aclType,aclsMap.get(aclType),denyAclsMap.get(aclType));
+ }
+
+ // Write the arguments
+ for (String name : arguments.keySet())
+ {
+ List<String> values = arguments.get(name);
+ writeField(out,name,values);
+ }
+
+ // Write the metadata, each in a field by itself
+ buildSolrParamsFromMetadata(out);
+
+ // These are unnecessary now in the case of non-solrcloud setups, because we overrode the SolrJ posting method to use multipart.
+ //writeField(out,LITERAL+"stream_size",String.valueOf(length));
+ //writeField(out,LITERAL+"stream_name",document.getFileName());
+
+ // General hint for Tika
+ if (document.getFileName() != null)
+ writeField(out,"resource.name",document.getFileName());
+
+ // Write the commitWithin parameter
+ if (commitWithin != null)
+ writeField(out,COMMITWITHIN_METADATA,commitWithin);
+
+ contentStreamUpdateRequest.setParams(out);
+
+ contentStreamUpdateRequest.addContentStream(new RepositoryDocumentStream(is,length,contentType,contentName));
+ }
+
/**
* builds the solr parameter maps for the update request.
* For each mapping expressed is applied the renaming for the metadata field name.
@@ -1044,6 +1175,34 @@ public class HttpPoster
}
}
+ private void buildSolrParamsFromMetadata(SolrInputDocument outputDocument) throws IOException
+ {
+ if (this.keepAllMetadata)
+ {
+ Iterator<String> iter = document.getFields();
+ while (iter.hasNext())
+ {
+ String fieldName = iter.next();
+ List<String> mappings = sourceTargets.get(fieldName);
+ if (mappings != null)
+ for (String newFieldName : mappings)
+ applySingleMapping(fieldName, outputDocument, newFieldName);
+ else // the fields not mentioned in the mapping are added only if we have set the keep all metadata=true.
+ applySingleMapping(fieldName, outputDocument, fieldName);
+ }
+ }
+ else
+ {
+ //don't keep all the metadata but only the ones in sourceTargets
+ for (String originalFieldName : sourceTargets.keySet())
+ {
+ List<String> mapping = sourceTargets.get(originalFieldName);
+ for (String newFieldName : mapping)
+ applySingleMapping(originalFieldName, outputDocument, newFieldName);
+ }
+ }
+ }
+
private void applySingleMapping(String originalFieldName, ModifiableSolrParams out, String newFieldName) throws IOException {
if(newFieldName != null && !newFieldName.isEmpty()) {
if (newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT))) {
@@ -1054,6 +1213,16 @@ public class HttpPoster
}
}
+ private void applySingleMapping(String originalFieldName, SolrInputDocument outputDocument, String newFieldName) throws IOException {
+ if(newFieldName != null && !newFieldName.isEmpty()) {
+ if (newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT))) {
+ newFieldName = ID_METADATA;
+ }
+ String[] values = document.getFieldAsStrings(originalFieldName);
+ outputDocument.addField( newFieldName, values );
+ }
+ }
+
public void finishUp()
throws InterruptedException, SolrServerException, IOException
{
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java?rev=1605929&r1=1605928&r2=1605929&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java Thu Jun 26 23:14:56 2014
@@ -111,6 +111,10 @@ public class SolrConfig
public static final String PARAM_INCLUDEDMIMETYPES = "Included mime types";
/** Excluded mime types */
public static final String PARAM_EXCLUDEDMIMETYPES="Excluded mime types";
+ /** Parameter describing the use of Extract Update handler */
+ public static final String PARAM_EXTRACTUPDATE = "Use extract update handler";
+ /** Optional content field (if not using extract update handler) */
+ public static final String PARAM_CONTENTFIELD = "Solr content field name";
/** Node describing an argument */
public static final String NODE_ARGUMENT = "argument";
/** Attribute with the argument name */
@@ -132,4 +136,5 @@ public class SolrConfig
*/
public static final String NODE_KEEPMETADATA = "keepAllMetadata";
+
}
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1605929&r1=1605928&r2=1605929&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java Thu Jun 26 23:14:56 2014
@@ -25,6 +25,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
+import java.util.HashSet;
import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
import org.apache.manifoldcf.agents.interfaces.IOutputNotifyActivity;
@@ -79,6 +81,17 @@ public class SolrConnector extends org.a
/** Excluded mime types */
protected Map<String,String> excludedMimeTypes = null;
+ // Attributes going into Solr
+ protected String idAttributeName = null;
+ protected String modifiedDateAttributeName = null;
+ protected String createdDateAttributeName = null;
+ protected String indexedDateAttributeName = null;
+ protected String fileNameAttributeName = null;
+ protected String mimeTypeAttributeName = null;
+ protected String contentAttributeName = null;
+ /** Use extractiing update handler? */
+ protected boolean useExtractUpdateHandler = true;
+
/** Whether or not to commit */
protected boolean doCommits = false;
@@ -162,6 +175,14 @@ public class SolrConnector extends org.a
includedMimeTypes = null;
excludedMimeTypesString = null;
excludedMimeTypes = null;
+ idAttributeName = null;
+ modifiedDateAttributeName = null;
+ createdDateAttributeName = null;
+ indexedDateAttributeName = null;
+ fileNameAttributeName = null;
+ mimeTypeAttributeName = null;
+ contentAttributeName = null;
+ useExtractUpdateHandler = true;
super.disconnect();
}
@@ -183,30 +204,42 @@ public class SolrConnector extends org.a
if (statusPath == null || statusPath.length() == 0)
statusPath = "";
- String idAttributeName = params.getParameter(SolrConfig.PARAM_IDFIELD);
+ idAttributeName = params.getParameter(SolrConfig.PARAM_IDFIELD);
if (idAttributeName == null || idAttributeName.length() == 0)
idAttributeName = "id";
- String modifiedDateAttributeName = params.getParameter(SolrConfig.PARAM_MODIFIEDDATEFIELD);
+ modifiedDateAttributeName = params.getParameter(SolrConfig.PARAM_MODIFIEDDATEFIELD);
if (modifiedDateAttributeName == null || modifiedDateAttributeName.length() == 0)
modifiedDateAttributeName = null;
- String createdDateAttributeName = params.getParameter(SolrConfig.PARAM_CREATEDDATEFIELD);
+ createdDateAttributeName = params.getParameter(SolrConfig.PARAM_CREATEDDATEFIELD);
if (createdDateAttributeName == null || createdDateAttributeName.length() == 0)
createdDateAttributeName = null;
- String indexedDateAttributeName = params.getParameter(SolrConfig.PARAM_INDEXEDDATEFIELD);
+ indexedDateAttributeName = params.getParameter(SolrConfig.PARAM_INDEXEDDATEFIELD);
if (indexedDateAttributeName == null || indexedDateAttributeName.length() == 0)
indexedDateAttributeName = null;
- String fileNameAttributeName = params.getParameter(SolrConfig.PARAM_FILENAMEFIELD);
+ fileNameAttributeName = params.getParameter(SolrConfig.PARAM_FILENAMEFIELD);
if (fileNameAttributeName == null || fileNameAttributeName.length() == 0)
fileNameAttributeName = null;
- String mimeTypeAttributeName = params.getParameter(SolrConfig.PARAM_MIMETYPEFIELD);
+ mimeTypeAttributeName = params.getParameter(SolrConfig.PARAM_MIMETYPEFIELD);
if (mimeTypeAttributeName == null || mimeTypeAttributeName.length() == 0)
mimeTypeAttributeName = null;
+ contentAttributeName = params.getParameter(SolrConfig.PARAM_CONTENTFIELD);
+ if (contentAttributeName == null || contentAttributeName.length() == 0)
+ contentAttributeName = null;
+
+ String useExtractUpdateHandlerValue = params.getParameter(SolrConfig.PARAM_EXTRACTUPDATE);
+ if (useExtractUpdateHandlerValue == null || useExtractUpdateHandlerValue.length() == 0)
+ useExtractUpdateHandler = true;
+ else
+ useExtractUpdateHandler = !useExtractUpdateHandlerValue.equals("false");
+ if (contentAttributeName == null && !useExtractUpdateHandler)
+ throw new ManifoldCFException("Content attribute name required for non-extract-update indexing");
+
String commits = params.getParameter(SolrConfig.PARAM_COMMITS);
if (commits == null || commits.length() == 0)
commits = "true";
@@ -222,6 +255,8 @@ public class SolrConnector extends org.a
maxDocumentLength = null;
else
maxDocumentLength = new Long(docMax);
+ if (maxDocumentLength == null && !useExtractUpdateHandler)
+ throw new ManifoldCFException("Maximum document length required for non-extract-update indexing");
includedMimeTypesString = params.getParameter(SolrConfig.PARAM_INCLUDEDMIMETYPES);
if (includedMimeTypesString == null || includedMimeTypesString.length() == 0)
@@ -313,8 +348,8 @@ public class SolrConnector extends org.a
updatePath,removePath,statusPath,realm,userID,password,
allowAttributeName,denyAttributeName,idAttributeName,
modifiedDateAttributeName,createdDateAttributeName,indexedDateAttributeName,
- fileNameAttributeName,mimeTypeAttributeName,
- keystoreManager,maxDocumentLength,commitWithin);
+ fileNameAttributeName,mimeTypeAttributeName,contentAttributeName,
+ keystoreManager,maxDocumentLength,commitWithin,useExtractUpdateHandler);
}
catch (NumberFormatException e)
@@ -368,8 +403,8 @@ public class SolrConnector extends org.a
updatePath,removePath,statusPath,
allowAttributeName,denyAttributeName,idAttributeName,
modifiedDateAttributeName,createdDateAttributeName,indexedDateAttributeName,
- fileNameAttributeName,mimeTypeAttributeName,
- maxDocumentLength,commitWithin);
+ fileNameAttributeName,mimeTypeAttributeName,contentAttributeName,
+ maxDocumentLength,commitWithin,useExtractUpdateHandler);
}
catch (NumberFormatException e)
@@ -460,152 +495,17 @@ public class SolrConnector extends org.a
throws ManifoldCFException, ServiceInterruption
{
getSession();
-
- StringBuilder sb = new StringBuilder();
-
- // All the arguments need to go into this string, since they affect ingestion.
- Map args = new HashMap();
- int i = 0;
- while (i < params.getChildCount())
- {
- ConfigNode node = params.getChild(i++);
- if (node.getType().equals(SolrConfig.NODE_ARGUMENT))
- {
- String attrName = node.getAttributeValue(SolrConfig.ATTRIBUTE_NAME);
- ArrayList list = (ArrayList)args.get(attrName);
- if (list == null)
- {
- list = new ArrayList();
- args.put(attrName,list);
- }
- list.add(node.getAttributeValue(SolrConfig.ATTRIBUTE_VALUE));
- }
- }
-
- String[] sortArray = new String[args.size()];
- Iterator iter = args.keySet().iterator();
- i = 0;
- while (iter.hasNext())
- {
- sortArray[i++] = (String)iter.next();
- }
-
- // Always use sorted order, because we need this to be comparable.
- java.util.Arrays.sort(sortArray);
-
- String[] fixedList = new String[2];
- ArrayList nameValues = new ArrayList();
- i = 0;
- while (i < sortArray.length)
- {
- String name = sortArray[i++];
- ArrayList values = (ArrayList)args.get(name);
- java.util.Collections.sort(values);
- int j = 0;
- while (j < values.size())
- {
- String value = (String)values.get(j++);
- fixedList[0] = name;
- fixedList[1] = value;
- StringBuilder pairBuffer = new StringBuilder();
- packFixedList(pairBuffer,fixedList,'=');
- nameValues.add(pairBuffer.toString());
- }
- }
-
- packList(sb,nameValues,'+');
-
- // Do the source/target pairs
- i = 0;
- Map<String, List<String>> sourceTargets = new HashMap<String, List<String>>();
- boolean keepAllMetadata = true;
- while (i < spec.getChildCount()) {
- SpecificationNode sn = spec.getChild(i++);
-
- if(sn.getType().equals(SolrConfig.NODE_KEEPMETADATA)) {
- String value = sn.getAttributeValue(SolrConfig.ATTRIBUTE_VALUE);
- keepAllMetadata = Boolean.parseBoolean(value);
- } else if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
- String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
- String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
-
- if (target == null) {
- target = "";
- }
- List<String> list = (List<String>)sourceTargets.get(source);
- if (list == null) {
- list = new ArrayList<String>();
- sourceTargets.put(source, list);
- }
- list.add(target);
- }
- }
-
- sortArray = new String[sourceTargets.size()];
- iter = sourceTargets.keySet().iterator();
- i = 0;
- while (iter.hasNext()) {
- sortArray[i++] = (String)iter.next();
- }
- java.util.Arrays.sort(sortArray);
-
- ArrayList sourceTargetsList = new ArrayList();
- i = 0;
- while (i < sortArray.length) {
- String source = sortArray[i++];
- List<String> values = (List<String>)sourceTargets.get(source);
- java.util.Collections.sort(values);
- int j = 0;
- while (j < values.size()) {
- String target = (String)values.get(j++);
- fixedList[0] = source;
- fixedList[1] = target;
- StringBuilder pairBuffer = new StringBuilder();
- packFixedList(pairBuffer,fixedList,'=');
- sourceTargetsList.add(pairBuffer.toString());
- }
- }
-
- packList(sb,sourceTargetsList,'+');
-
- // Keep all metadata flag
- if (keepAllMetadata)
- sb.append('+');
- else
- sb.append('-');
+ SpecPacker sp = new SpecPacker(spec);
+ return sp.toPackedString();
+ }
- // Here, append things which we have no intention of unpacking. This includes stuff that comes from
- // the configuration information, for instance.
-
- if (maxDocumentLength != null || includedMimeTypesString != null || excludedMimeTypesString != null)
- {
- // Length limitation. We pack this because when it is changed we want to be sure we get any previously excluded documents.
- if (maxDocumentLength != null)
- {
- sb.append('+');
- pack(sb,maxDocumentLength.toString(),'+');
- }
- else
- sb.append('-');
- // Included mime types
- if (includedMimeTypesString != null)
- {
- sb.append('+');
- pack(sb,includedMimeTypesString,'+');
- }
- else
- sb.append('-');
- // Excluded mime types
- if (excludedMimeTypesString != null)
- {
- sb.append('+');
- pack(sb,excludedMimeTypesString,'+');
- }
- else
- sb.append('-');
- }
-
- return sb.toString();
+ private final static Set<String> acceptableMimeTypes = new HashSet<String>();
+ static
+ {
+ acceptableMimeTypes.add("text/plain;charset=utf-8");
+ acceptableMimeTypes.add("text/plain;charset=ascii");
+ acceptableMimeTypes.add("text/plain;charset=us-ascii");
+ acceptableMimeTypes.add("text/plain");
}
/** Detect if a mime type is indexable or not. This method is used by participating repository connectors to pre-filter the number of
@@ -618,11 +518,15 @@ public class SolrConnector extends org.a
throws ManifoldCFException, ServiceInterruption
{
getSession();
- if (includedMimeTypes != null && includedMimeTypes.get(mimeType) == null)
- return false;
- if (excludedMimeTypes != null && excludedMimeTypes.get(mimeType) != null)
- return false;
- return super.checkMimeTypeIndexable(outputDescription,mimeType);
+ if (useExtractUpdateHandler)
+ {
+ if (includedMimeTypes != null && includedMimeTypes.get(mimeType) == null)
+ return false;
+ if (excludedMimeTypes != null && excludedMimeTypes.get(mimeType) != null)
+ return false;
+ return super.checkMimeTypeIndexable(outputDescription,mimeType);
+ }
+ return acceptableMimeTypes.contains(mimeType.toLowerCase(Locale.ROOT));
}
/** Pre-determine whether a document's length is indexable by this connector. This method is used by participating repository connectors
@@ -658,59 +562,13 @@ public class SolrConnector extends org.a
public int addOrReplaceDocument(String documentURI, String outputDescription, RepositoryDocument document, String authorityNameString, IOutputAddActivity activities)
throws ManifoldCFException, ServiceInterruption
{
- // Build the argument map we'll send.
- Map args = new HashMap();
- Map<String, List<String>> sourceTargets = new HashMap<String, List<String>>();
- int index = 0;
- ArrayList nameValues = new ArrayList();
- index = unpackList(nameValues,outputDescription,index,'+');
- ArrayList sts = new ArrayList();
- index = unpackList(sts,outputDescription,index,'+');
- // extract keep all metadata Flag
- boolean keepAllMetadata = true;
- if (index < outputDescription.length())
- {
- keepAllMetadata = (outputDescription.charAt(index++) == '+');
- }
- String[] fixedBuffer = new String[2];
-
- // Do the name/value pairs
- int i = 0;
- while (i < nameValues.size())
- {
- String x = (String)nameValues.get(i++);
- unpackFixedList(fixedBuffer,x,0,'=');
- String attrName = fixedBuffer[0];
- ArrayList list = (ArrayList)args.get(attrName);
- if (list == null)
- {
- list = new ArrayList();
- args.put(attrName,list);
- }
- list.add(fixedBuffer[1]);
- }
-
- // Do the source/target pairs
- i = 0;
- while (i < sts.size()) {
- String x = (String)sts.get(i++);
- unpackFixedList(fixedBuffer,x,0,'=');
- String source = fixedBuffer[0];
- String target = fixedBuffer[1];
- List<String> list = (List<String>)sourceTargets.get(source);
- if (list == null) {
- list = new ArrayList<String>();
- sourceTargets.put(source, list);
- }
- list.add(target);
- }
-
+ SpecPacker sp = new SpecPacker(outputDescription);
// Establish a session
getSession();
// Now, go off and call the ingest API.
- if (poster.indexPost(documentURI,document,args,sourceTargets,keepAllMetadata,authorityNameString,activities))
+ if (poster.indexPost(documentURI,document,sp.getArgs(),sp.getMappings(),sp.keepAllMetadata(),authorityNameString,activities))
return DOCUMENTSTATUS_ACCEPTED;
return DOCUMENTSTATUS_REJECTED;
}
@@ -966,6 +824,20 @@ public class SolrConnector extends org.a
" editconnection.maxdocumentlength.focus();\n"+
" return false;\n"+
" }\n"+
+" if (editconnection.maxdocumentlength.value == \"\" && (editconnection.extractupdate.value != \"true\" || (editconnection.extractupdate.checked == false)))\n"+
+" {\n"+
+" alert(\""+Messages.getBodyJavascriptString(locale,"SolrConnector.MaximumDocumentLengthRequiredUnlessExtractingUpdateHandler")+"\");\n"+
+" SelectTab(\""+Messages.getBodyJavascriptString(locale,"SolrConnector.Documents")+"\");\n"+
+" editconnection.maxdocumentlength.focus();\n"+
+" return false;\n"+
+" }\n"+
+" if (editconnection.contentfield.value == \"\" && (editconnection.extractupdate.value != \"true\" || (editconnection.extractupdate.checked == false)))\n"+
+" {\n"+
+" alert(\""+Messages.getBodyJavascriptString(locale,"SolrConnector.ContentFieldNameRequiredUnlessExtractingUpdateHandler")+"\");\n"+
+" SelectTab(\""+Messages.getBodyJavascriptString(locale,"SolrConnector.Schema")+"\");\n"+
+" editconnection.contentfield.focus();\n"+
+" return false;\n"+
+" }\n"+
" if (editconnection.commitwithin.value != \"\" && !isInteger(editconnection.commitwithin.value))\n"+
" {\n"+
" alert(\""+Messages.getBodyJavascriptString(locale,"SolrConnector.CommitWithinValueMustBeAnInteger")+"\");\n"+
@@ -1147,7 +1019,15 @@ public class SolrConnector extends org.a
String mimeTypeField = parameters.getParameter(SolrConfig.PARAM_MIMETYPEFIELD);
if (mimeTypeField == null)
mimeTypeField = "";
-
+
+ String contentField = parameters.getParameter(SolrConfig.PARAM_CONTENTFIELD);
+ if (contentField == null)
+ contentField = "";
+
+ String useExtractUpdate = parameters.getParameter(SolrConfig.PARAM_EXTRACTUPDATE);
+ if (useExtractUpdate == null || useExtractUpdate.length() == 0)
+ useExtractUpdate = "true";
+
String realm = parameters.getParameter(SolrConfig.PARAM_REALM);
if (realm == null)
realm = "";
@@ -1602,6 +1482,32 @@ public class SolrConnector extends org.a
" <input name=\"mimetypefield\" type=\"text\" size=\"32\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(mimeTypeField)+"\"/>\n"+
" </td>\n"+
" </tr>\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"SolrConnector.UseExtractUpdateHandler") + "</nobr></td>\n"+
+" <td class=\"value\">\n"+
+" <input name=\"extractupdatepresent\" type=\"hidden\" value=\"true\"/>\n"
+ );
+ if (!useExtractUpdate.equals("false"))
+ {
+ out.print(
+" <input name=\"extractupdate\" type=\"checkbox\" value=\"true\" checked=\"true\"/>\n"
+ );
+ }
+ else
+ {
+ out.print(
+" <input name=\"extractupdate\" type=\"checkbox\" value=\"true\"/>\n"
+ );
+ }
+ out.print(
+" </td>\n"+
+" </tr>\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"SolrConnector.ContentFieldName") + "</nobr></td>\n"+
+" <td class=\"value\">\n"+
+" <input name=\"contentfield\" type=\"text\" size=\"32\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(contentField)+"\"/>\n"+
+" </td>\n"+
+" </tr>\n"+
"</table>\n"
);
}
@@ -1613,7 +1519,10 @@ public class SolrConnector extends org.a
"<input type=\"hidden\" name=\"createddatefield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(createdDateField)+"\"/>\n"+
"<input type=\"hidden\" name=\"indexeddatefield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(indexedDateField)+"\"/>\n"+
"<input type=\"hidden\" name=\"filenamefield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(fileNameField)+"\"/>\n"+
-"<input type=\"hidden\" name=\"mimetypefield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(mimeTypeField)+"\"/>\n"
+"<input type=\"hidden\" name=\"mimetypefield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(mimeTypeField)+"\"/>\n"+
+"<input type=\"hidden\" name=\"contentfield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(contentField)+"\"/>\n"+
+"<input type=\"hidden\" name=\"extractupdate\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(useExtractUpdate)+"\"/>\n"+
+"<input name=\"extractupdatepresent\" type=\"hidden\" value=\"true\"/>\n"
);
}
@@ -1913,6 +1822,19 @@ public class SolrConnector extends org.a
if (mimeTypeField != null)
parameters.setParameter(SolrConfig.PARAM_MIMETYPEFIELD,mimeTypeField);
+ String contentField = variableContext.getParameter("contentfield");
+ if (contentField != null)
+ parameters.setParameter(SolrConfig.PARAM_CONTENTFIELD,contentField);
+
+ String extractUpdatePresent = variableContext.getParameter("extractupdatepresent");
+ if (extractUpdatePresent != null)
+ {
+ String extractUpdate = variableContext.getParameter("extractupdate");
+ if (extractUpdate == null || extractUpdate.length() == 0)
+ extractUpdate = "false";
+ parameters.setParameter(SolrConfig.PARAM_EXTRACTUPDATE,extractUpdate);
+ }
+
String realm = variableContext.getParameter("realm");
if (realm != null)
parameters.setParameter(SolrConfig.PARAM_REALM,realm);
@@ -2646,4 +2568,285 @@ public class SolrConnector extends org.a
}
+ /** This class handles Solr connector version string packing/unpacking/interpretation.
+ */
+ protected class SpecPacker {
+
+ /** Arguments, from configuration */
+ private final Map<String,List<String>> args = new HashMap<String,List<String>>();
+ /** Source/targets from specification */
+ private final Map<String, List<String>> sourceTargets = new HashMap<String, List<String>>();
+ /** Keep all metadata flag, from specification */
+ private final boolean keepAllMetadata;
+
+ public SpecPacker(Specification spec) {
+
+ // Process arguments
+ for (int i = 0; i < params.getChildCount(); i++)
+ {
+ ConfigNode node = params.getChild(i);
+ if (node.getType().equals(SolrConfig.NODE_ARGUMENT))
+ {
+ String attrName = node.getAttributeValue(SolrConfig.ATTRIBUTE_NAME);
+ List<String> list = args.get(attrName);
+ if (list == null)
+ {
+ list = new ArrayList<String>();
+ args.put(attrName,list);
+ }
+ list.add(node.getAttributeValue(SolrConfig.ATTRIBUTE_VALUE));
+ }
+ }
+
+ // Do the source/target pairs
+ boolean keepAllMetadata = true;
+ for (int i = 0; i < spec.getChildCount(); i++)
+ {
+ SpecificationNode sn = spec.getChild(i);
+
+ if(sn.getType().equals(SolrConfig.NODE_KEEPMETADATA)) {
+ String value = sn.getAttributeValue(SolrConfig.ATTRIBUTE_VALUE);
+ keepAllMetadata = Boolean.parseBoolean(value);
+ } else if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
+ String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
+ String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
+
+ if (target == null) {
+ target = "";
+ }
+ List<String> list = sourceTargets.get(source);
+ if (list == null) {
+ list = new ArrayList<String>();
+ sourceTargets.put(source, list);
+ }
+ list.add(target);
+ }
+ }
+ this.keepAllMetadata = keepAllMetadata;
+
+ }
+
+ /** Packed string parser.
+ * This method unpacks a packed version string, and makes the formerly packed data available for use.
+ * Note that it is actually *not* a requirement for this method to do the unpacking; that can happen "on demand"
+ * for performance, if deemed helpful.
+ */
+ public SpecPacker(String packedString) {
+ // Build the argument map we'll send.
+ int index = 0;
+ List<String> nameValues = new ArrayList<String>();
+ index = unpackList(nameValues,packedString,index,'+');
+ List<String> sts = new ArrayList<String>();
+ index = unpackList(sts,packedString,index,'+');
+ // extract keep all metadata Flag
+ boolean keepAllMetadata = true;
+ if (index < packedString.length())
+ {
+ keepAllMetadata = (packedString.charAt(index++) == '+');
+ }
+ this.keepAllMetadata = keepAllMetadata;
+
+
+ String[] fixedBuffer = new String[2];
+
+ // Do the name/value pairs
+ for (String x : nameValues)
+ {
+ unpackFixedList(fixedBuffer,x,0,'=');
+ String attrName = fixedBuffer[0];
+ List<String> list = args.get(attrName);
+ if (list == null)
+ {
+ list = new ArrayList<String>();
+ args.put(attrName,list);
+ }
+ list.add(fixedBuffer[1]);
+ }
+
+ // Do the source/target pairs
+ for (String x : sts)
+ {
+ unpackFixedList(fixedBuffer,x,0,'=');
+ String source = fixedBuffer[0];
+ String target = fixedBuffer[1];
+ List<String> list = sourceTargets.get(source);
+ if (list == null) {
+ list = new ArrayList<String>();
+ sourceTargets.put(source, list);
+ }
+ list.add(target);
+ }
+
+ }
+
+ public String toPackedString() {
+ StringBuilder sb = new StringBuilder();
+ String[] sortArray = new String[args.size()];
+ Iterator<String> iter = args.keySet().iterator();
+ int i = 0;
+ while (iter.hasNext())
+ {
+ sortArray[i++] = iter.next();
+ }
+
+ // Always use sorted order, because we need this to be comparable.
+ java.util.Arrays.sort(sortArray);
+
+ String[] fixedList = new String[2];
+ List<String> nameValues = new ArrayList<String>();
+ for (int k = 0; k < sortArray.length; k++)
+ {
+ String name = sortArray[k];
+ List<String> values = args.get(name);
+ java.util.Collections.sort(values);
+ for (String value : values)
+ {
+ fixedList[0] = name;
+ fixedList[1] = value;
+ StringBuilder pairBuffer = new StringBuilder();
+ packFixedList(pairBuffer,fixedList,'=');
+ nameValues.add(pairBuffer.toString());
+ }
+ }
+
+ packList(sb,nameValues,'+');
+
+ // Do the source/target pairs
+ sortArray = new String[sourceTargets.size()];
+ iter = sourceTargets.keySet().iterator();
+ i = 0;
+ while (iter.hasNext()) {
+ sortArray[i++] = iter.next();
+ }
+ java.util.Arrays.sort(sortArray);
+
+ List<String> sourceTargetsList = new ArrayList<String>();
+ for (int k = 0; k < sortArray.length; k++)
+ {
+ String source = sortArray[k];
+ List<String> values = sourceTargets.get(source);
+ java.util.Collections.sort(values);
+ for (String target : values)
+ {
+ fixedList[0] = source;
+ fixedList[1] = target;
+ StringBuilder pairBuffer = new StringBuilder();
+ packFixedList(pairBuffer,fixedList,'=');
+ sourceTargetsList.add(pairBuffer.toString());
+ }
+ }
+
+ packList(sb,sourceTargetsList,'+');
+
+ // Keep all metadata flag
+ if (keepAllMetadata)
+ sb.append('+');
+ else
+ sb.append('-');
+
+ // Here, append things which we have no intention of unpacking. This includes stuff that comes from
+ // the configuration information, for instance.
+
+ if (idAttributeName != null)
+ {
+ sb.append('+');
+ pack(sb,idAttributeName,'+');
+ }
+ else
+ sb.append('-');
+
+ if (modifiedDateAttributeName != null)
+ {
+ sb.append('+');
+ pack(sb,modifiedDateAttributeName,'+');
+ }
+ else
+ sb.append('-');
+
+ if (createdDateAttributeName != null)
+ {
+ sb.append('+');
+ pack(sb,createdDateAttributeName,'+');
+ }
+ else
+ sb.append('-');
+
+ if (indexedDateAttributeName != null)
+ {
+ sb.append('+');
+ pack(sb,indexedDateAttributeName,'+');
+ }
+ else
+ sb.append('-');
+
+ if (fileNameAttributeName != null)
+ {
+ sb.append('+');
+ pack(sb,fileNameAttributeName,'+');
+ }
+ else
+ sb.append('-');
+
+ if (mimeTypeAttributeName != null)
+ {
+ sb.append('+');
+ pack(sb,mimeTypeAttributeName,'+');
+ }
+ else
+ sb.append('-');
+
+ if (contentAttributeName != null)
+ {
+ sb.append('+');
+ pack(sb,contentAttributeName,'+');
+ }
+ else
+ sb.append('-');
+
+ if (useExtractUpdateHandler)
+ sb.append('+');
+ else
+ sb.append('-');
+
+ // Length limitation. We pack this because when it is changed we want to be sure we get any previously excluded documents.
+ if (maxDocumentLength != null)
+ {
+ sb.append('+');
+ pack(sb,maxDocumentLength.toString(),'+');
+ }
+ else
+ sb.append('-');
+ // Included mime types
+ if (includedMimeTypesString != null)
+ {
+ sb.append('+');
+ pack(sb,includedMimeTypesString,'+');
+ }
+ else
+ sb.append('-');
+ // Excluded mime types
+ if (excludedMimeTypesString != null)
+ {
+ sb.append('+');
+ pack(sb,excludedMimeTypesString,'+');
+ }
+ else
+ sb.append('-');
+
+ return sb.toString();
+ }
+
+ public Map<String,List<String>> getArgs() {
+ return args;
+ }
+
+ public Map<String,List<String>> getMappings() {
+ return sourceTargets;
+ }
+
+ public boolean keepAllMetadata() {
+ return keepAllMetadata;
+ }
+ }
+
}
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties?rev=1605929&r1=1605928&r2=1605929&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties Thu Jun 26 23:14:56 2014
@@ -15,6 +15,7 @@
SolrConnector.Arguments=Arguments
SolrConnector.KeepAllMetadata=Keep all metadata:
+SolrConnector.UseExtractUpdateHandler=Use the Extract Update Handler:
SolrConnector.Arguments2=Arguments:
SolrConnector.Commits=Commits
SolrConnector.Documents=Documents
@@ -57,6 +58,7 @@ SolrConnector.CreatedDateFieldName=Creat
SolrConnector.IndexedDateFieldName=Indexed date field name:
SolrConnector.FileNameFieldName=File name field name:
SolrConnector.MimeTypeFieldName=Mime type field name:
+SolrConnector.ContentFieldName=Content field name:
SolrConnector.MaximumDocumentLength=Maximum document length:
SolrConnector.IncludedMimeTypes=Included mime types:
SolrConnector.ExcludedMimeTypes=Excluded mime types:
@@ -94,6 +96,8 @@ SolrConnector.MaximumDocumentLengthMustB
SolrConnector.CommitWithinValueMustBeAnInteger=Commit-within value must be an integer
SolrConnector.MaximumDocumentLengthMustBeAnInteger=Maximum document length must be an integer
SolrConnector.ArgumentNameCannotBeAnEmptyString=Argument name cannot be an empty string
+SolrConnector.MaximumDocumentLengthRequiredUnlessExtractingUpdateHandler=Maximum document length required unless using extract update handler
+SolrConnector.ContentFieldNameRequiredUnlessExtractingUpdateHandler=Content field name required unless using extract update handler
SolrConnector.DeleteCert=Delete cert
SolrConnector.Delete=Delete
SolrConnector.DeleteArgument=Delete argument #
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties?rev=1605929&r1=1605928&r2=1605929&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties Thu Jun 26 23:14:56 2014
@@ -15,6 +15,7 @@
SolrConnector.Arguments=å¼æ°
SolrConnector.KeepAllMetadata=Keep all metadata:
+SolrConnector.UseExtractUpdateHandler=Use the Extract Update Handler:
SolrConnector.Arguments2=å¼æ°ï¼
SolrConnector.Commits=ã³ããã
SolrConnector.Documents=ã³ã³ãã³ã
@@ -57,6 +58,7 @@ SolrConnector.CreatedDateFieldName=�
SolrConnector.IndexedDateFieldName=Indexed date field name:
SolrConnector.FileNameFieldName=ãã¡ã¤ã«å称ãã£ã¼ã«ãåï¼
SolrConnector.MimeTypeFieldName=MIMEã¿ã¤ããã£ã¼ã«ãåï¼
+SolrConnector.ContentFieldName=Content field name:
SolrConnector.MaximumDocumentLength=æ大ã³ã³ãã³ãé·ï¼
SolrConnector.IncludedMimeTypes=å«ãMIMEã¿ã¤ãï¼
SolrConnector.ExcludedMimeTypes=é¤å¤ããMIMEã¿ã¤ãï¼
@@ -94,6 +96,8 @@ SolrConnector.MaximumDocumentLengthMustB
SolrConnector.CommitWithinValueMustBeAnInteger=Commit-withinå¤ã«ã¯æ´æ°ãå
¥åãã¦ãã ãã
SolrConnector.MaximumDocumentLengthMustBeAnInteger=æ大ã³ã³ãã³ãé·ãã«ã¯æ´æ°ãå
¥åãã¦ãã ãã
SolrConnector.ArgumentNameCannotBeAnEmptyString=å¼æ°åãå
¥åãã¦ãã ãã
+SolrConnector.MaximumDocumentLengthRequiredUnlessExtractingUpdateHandler=Maximum document length required unless using extract update handler
+SolrConnector.ContentFieldNameRequiredUnlessExtractingUpdateHandler=Content field name required unless using extract update handler
SolrConnector.DeleteCert=証ææ¸ãåé¤
SolrConnector.Delete=åé¤
SolrConnector.DeleteArgument=å¼æ°ãåé¤ #
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties?rev=1605929&r1=1605928&r2=1605929&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties Thu Jun 26 23:14:56 2014
@@ -15,6 +15,7 @@
SolrConnector.Arguments=åæ°
SolrConnector.KeepAllMetadata=ä¿æææå
æ°æ®:
+SolrConnector.UseExtractUpdateHandler=Use the Extract Update Handler:
SolrConnector.Arguments2=åæ°:
SolrConnector.Commits=æ交
SolrConnector.Documents=ææ¡£
@@ -57,6 +58,7 @@ SolrConnector.CreatedDateFieldName=ç�
SolrConnector.IndexedDateFieldName=ç´¢å¼åçæ¥æå段å:
SolrConnector.FileNameFieldName=æ件åå段å:
SolrConnector.MimeTypeFieldName=MIMEç±»åå段å:
+SolrConnector.ContentFieldName=Content field name:
SolrConnector.MaximumDocumentLength=æ大ææ¡£é¿åº¦:
SolrConnector.IncludedMimeTypes=æå
å«çMIMEç±»å:
SolrConnector.ExcludedMimeTypes=被æé¤çMIMEç±»å:
@@ -94,6 +96,8 @@ SolrConnector.MaximumDocumentLengthMustB
SolrConnector.CommitWithinValueMustBeAnInteger=CommitWithinå¼å¿
须为æ´æ°
SolrConnector.MaximumDocumentLengthMustBeAnInteger=æ大ææ¡£é¿åº¦å¿
须为æ´æ°
SolrConnector.ArgumentNameCannotBeAnEmptyString=请è¾å
¥åæ°å
+SolrConnector.MaximumDocumentLengthRequiredUnlessExtractingUpdateHandler=Maximum document length required unless using extract update handler
+SolrConnector.ContentFieldNameRequiredUnlessExtractingUpdateHandler=Content field name required unless using extract update handler
SolrConnector.DeleteCert=å é¤è¯ä¹¦
SolrConnector.Delete=å é¤
SolrConnector.DeleteArgument=å é¤åæ° #