You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/06/25 16:26:05 UTC
svn commit: r1605419 - in
/manifoldcf/branches/CONNECTORS-981/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr:
SolrConfig.java SolrConnector.java
Author: kwright
Date: Wed Jun 25 14:26:05 2014
New Revision: 1605419
URL: http://svn.apache.org/r1605419
Log:
Include solr indexing parameters in version strings
Modified:
manifoldcf/branches/CONNECTORS-981/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
manifoldcf/branches/CONNECTORS-981/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Modified: manifoldcf/branches/CONNECTORS-981/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-981/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java?rev=1605419&r1=1605418&r2=1605419&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-981/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java (original)
+++ manifoldcf/branches/CONNECTORS-981/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java Wed Jun 25 14:26:05 2014
@@ -111,6 +111,10 @@ public class SolrConfig
public static final String PARAM_INCLUDEDMIMETYPES = "Included mime types";
/** Excluded mime types */
public static final String PARAM_EXCLUDEDMIMETYPES="Excluded mime types";
+ /** Parameter describing the use of Extract Update handler */
+ public static final String PARAM_EXTRACTUPDATE = "Use extract update handler";
+ /** Optional content field (if not using extract update handler) */
+ public static final String PARAM_CONTENTFIELD = "Solr content field name";
/** Node describing an argument */
public static final String NODE_ARGUMENT = "argument";
/** Attribute with the argument name */
@@ -132,9 +136,5 @@ public class SolrConfig
*/
public static final String NODE_KEEPMETADATA = "keepAllMetadata";
- /**
- * Node describing the use of Extract Update handler
- */
- public static final String NODE_EXTRACTUPDATE = "useExtractUpdateHandler";
}
Modified: manifoldcf/branches/CONNECTORS-981/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-981/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1605419&r1=1605418&r2=1605419&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-981/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-981/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java Wed Jun 25 14:26:05 2014
@@ -80,6 +80,15 @@ public class SolrConnector extends org.a
protected String excludedMimeTypesString = null;
/** Excluded mime types */
protected Map<String,String> excludedMimeTypes = null;
+
+ // Attributes going into Solr
+ protected String idAttributeName = null;
+ protected String modifiedDateAttributeName = null;
+ protected String createdDateAttributeName = null;
+ protected String indexedDateAttributeName = null;
+ protected String fileNameAttributeName = null;
+ protected String mimeTypeAttributeName = null;
+ protected String contentAttributeName = null;
/** Use extractiing update handler? */
protected boolean useExtractUpdateHandler = true;
@@ -166,6 +175,13 @@ public class SolrConnector extends org.a
includedMimeTypes = null;
excludedMimeTypesString = null;
excludedMimeTypes = null;
+ idAttributeName = null;
+ modifiedDateAttributeName = null;
+ createdDateAttributeName = null;
+ indexedDateAttributeName = null;
+ fileNameAttributeName = null;
+ mimeTypeAttributeName = null;
+ contentAttributeName = null;
useExtractUpdateHandler = true;
super.disconnect();
}
@@ -188,33 +204,42 @@ public class SolrConnector extends org.a
if (statusPath == null || statusPath.length() == 0)
statusPath = "";
- String idAttributeName = params.getParameter(SolrConfig.PARAM_IDFIELD);
+ idAttributeName = params.getParameter(SolrConfig.PARAM_IDFIELD);
if (idAttributeName == null || idAttributeName.length() == 0)
idAttributeName = "id";
- String modifiedDateAttributeName = params.getParameter(SolrConfig.PARAM_MODIFIEDDATEFIELD);
+ modifiedDateAttributeName = params.getParameter(SolrConfig.PARAM_MODIFIEDDATEFIELD);
if (modifiedDateAttributeName == null || modifiedDateAttributeName.length() == 0)
modifiedDateAttributeName = null;
- String createdDateAttributeName = params.getParameter(SolrConfig.PARAM_CREATEDDATEFIELD);
+ createdDateAttributeName = params.getParameter(SolrConfig.PARAM_CREATEDDATEFIELD);
if (createdDateAttributeName == null || createdDateAttributeName.length() == 0)
createdDateAttributeName = null;
- String indexedDateAttributeName = params.getParameter(SolrConfig.PARAM_INDEXEDDATEFIELD);
+ indexedDateAttributeName = params.getParameter(SolrConfig.PARAM_INDEXEDDATEFIELD);
if (indexedDateAttributeName == null || indexedDateAttributeName.length() == 0)
indexedDateAttributeName = null;
- String fileNameAttributeName = params.getParameter(SolrConfig.PARAM_FILENAMEFIELD);
+ fileNameAttributeName = params.getParameter(SolrConfig.PARAM_FILENAMEFIELD);
if (fileNameAttributeName == null || fileNameAttributeName.length() == 0)
fileNameAttributeName = null;
- String mimeTypeAttributeName = params.getParameter(SolrConfig.PARAM_MIMETYPEFIELD);
+ mimeTypeAttributeName = params.getParameter(SolrConfig.PARAM_MIMETYPEFIELD);
if (mimeTypeAttributeName == null || mimeTypeAttributeName.length() == 0)
mimeTypeAttributeName = null;
- String contentAttributeName = "content"; // ??? -- should be settable
- useExtractUpdateHandler = true; // ???
-
+ contentAttributeName = params.getParameter(SolrConfig.PARAM_CONTENTFIELD);
+ if (contentAttributeName == null || contentAttributeName.length() == 0)
+ contentAttributeName = null;
+
+ String useExtractUpdateHandlerValue = params.getParameter(SolrConfig.PARAM_EXTRACTUPDATE);
+ if (useExtractUpdateHandlerValue == null || useExtractUpdateHandlerValue.length() == 0)
+ useExtractUpdateHandler = true;
+ else
+ useExtractUpdateHandler = !useExtractUpdateHandlerValue.equals("false");
+ if (contentAttributeName == null && !useExtractUpdateHandler)
+ throw new ManifoldCFException("Content attribute name required for non-extract-update indexing");
+
String commits = params.getParameter(SolrConfig.PARAM_COMMITS);
if (commits == null || commits.length() == 0)
commits = "true";
@@ -230,6 +255,8 @@ public class SolrConnector extends org.a
maxDocumentLength = null;
else
maxDocumentLength = new Long(docMax);
+ if (maxDocumentLength == null && !useExtractUpdateHandler)
+ throw new ManifoldCFException("Maximum document length required for non-extract-update indexing");
includedMimeTypesString = params.getParameter(SolrConfig.PARAM_INCLUDEDMIMETYPES);
if (includedMimeTypesString == null || includedMimeTypesString.length() == 0)
@@ -2655,35 +2682,93 @@ public class SolrConnector extends org.a
// Here, append things which we have no intention of unpacking. This includes stuff that comes from
// the configuration information, for instance.
+
+ if (idAttributeName != null)
+ {
+ sb.append('+');
+ pack(sb,idAttributeName,'+');
+ }
+ else
+ sb.append('-');
+
+ if (modifiedDateAttributeName != null)
+ {
+ sb.append('+');
+ pack(sb,modifiedDateAttributeName,'+');
+ }
+ else
+ sb.append('-');
- if (maxDocumentLength != null || includedMimeTypesString != null || excludedMimeTypesString != null)
+ if (createdDateAttributeName != null)
{
- // Length limitation. We pack this because when it is changed we want to be sure we get any previously excluded documents.
- if (maxDocumentLength != null)
- {
sb.append('+');
- pack(sb,maxDocumentLength.toString(),'+');
- }
- else
- sb.append('-');
- // Included mime types
- if (includedMimeTypesString != null)
- {
+ pack(sb,createdDateAttributeName,'+');
+ }
+ else
+ sb.append('-');
+
+ if (indexedDateAttributeName != null)
+ {
sb.append('+');
- pack(sb,includedMimeTypesString,'+');
- }
- else
- sb.append('-');
- // Excluded mime types
- if (excludedMimeTypesString != null)
- {
+ pack(sb,indexedDateAttributeName,'+');
+ }
+ else
+ sb.append('-');
+
+ if (fileNameAttributeName != null)
+ {
sb.append('+');
- pack(sb,excludedMimeTypesString,'+');
- }
- else
- sb.append('-');
+ pack(sb,fileNameAttributeName,'+');
}
-
+ else
+ sb.append('-');
+
+ if (mimeTypeAttributeName != null)
+ {
+ sb.append('+');
+ pack(sb,mimeTypeAttributeName,'+');
+ }
+ else
+ sb.append('-');
+
+ if (contentAttributeName != null)
+ {
+ sb.append('+');
+ pack(sb,contentAttributeName,'+');
+ }
+ else
+ sb.append('-');
+
+ if (useExtractUpdateHandler)
+ sb.append('+');
+ else
+ sb.append('-');
+
+ // Length limitation. We pack this because when it is changed we want to be sure we get any previously excluded documents.
+ if (maxDocumentLength != null)
+ {
+ sb.append('+');
+ pack(sb,maxDocumentLength.toString(),'+');
+ }
+ else
+ sb.append('-');
+ // Included mime types
+ if (includedMimeTypesString != null)
+ {
+ sb.append('+');
+ pack(sb,includedMimeTypesString,'+');
+ }
+ else
+ sb.append('-');
+ // Excluded mime types
+ if (excludedMimeTypesString != null)
+ {
+ sb.append('+');
+ pack(sb,excludedMimeTypesString,'+');
+ }
+ else
+ sb.append('-');
+
return sb.toString();
}