You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/03/04 23:16:09 UTC
svn commit: r1452563 - in
/manifoldcf/trunk/connectors/solr/connector/src/main:
java/org/apache/manifoldcf/agents/output/solr/
native2ascii/org/apache/manifoldcf/agents/output/solr/
Author: kwright
Date: Mon Mar 4 22:16:08 2013
New Revision: 1452563
URL: http://svn.apache.org/r1452563
Log:
Add common field support to Solr connector. Part of CONNECTORS-657.
Modified:
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1452563&r1=1452562&r2=1452563&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java Mon Mar 4 22:16:08 2013
@@ -103,6 +103,10 @@ public class HttpPoster
private String allowAttributeName;
private String denyAttributeName;
private String idAttributeName;
+ private String modifiedDateAttributeName;
+ private String createdDateAttributeName;
+ private String fileNameAttributeName;
+ private String mimeTypeAttributeName;
// Document max length
private Long maxDocumentLength;
@@ -125,6 +129,8 @@ public class HttpPoster
int zkClientTimeout, int zkConnectTimeout,
String updatePath, String removePath, String statusPath,
String allowAttributeName, String denyAttributeName, String idAttributeName,
+ String modifiedDateAttributeName, String createdDateAttributeName,
+ String fileNameAttributeName, String mimeTypeAttributeName,
Long maxDocumentLength,
String commitWithin)
throws ManifoldCFException
@@ -139,6 +145,10 @@ public class HttpPoster
this.allowAttributeName = allowAttributeName;
this.denyAttributeName = denyAttributeName;
this.idAttributeName = idAttributeName;
+ this.modifiedDateAttributeName = modifiedDateAttributeName;
+ this.createdDateAttributeName = createdDateAttributeName;
+ this.fileNameAttributeName = fileNameAttributeName;
+ this.mimeTypeAttributeName = mimeTypeAttributeName;
this.maxDocumentLength = maxDocumentLength;
@@ -164,6 +174,8 @@ public class HttpPoster
String updatePath, String removePath, String statusPath,
String realm, String userID, String password,
String allowAttributeName, String denyAttributeName, String idAttributeName,
+ String modifiedDateAttributeName, String createdDateAttributeName,
+ String fileNameAttributeName, String mimeTypeAttributeName,
IKeystoreManager keystoreManager, Long maxDocumentLength,
String commitWithin)
throws ManifoldCFException
@@ -178,6 +190,10 @@ public class HttpPoster
this.allowAttributeName = allowAttributeName;
this.denyAttributeName = denyAttributeName;
this.idAttributeName = idAttributeName;
+ this.modifiedDateAttributeName = modifiedDateAttributeName;
+ this.createdDateAttributeName = createdDateAttributeName;
+ this.fileNameAttributeName = fileNameAttributeName;
+ this.mimeTypeAttributeName = mimeTypeAttributeName;
this.maxDocumentLength = maxDocumentLength;
@@ -782,7 +798,34 @@ public class HttpPoster
// Write the id field
writeField(out,LITERAL+idAttributeName,documentURI);
-
+ // Write the rest of the attributes
+ if (modifiedDateAttributeName != null)
+ {
+ Date date = document.getModifiedDate();
+ if (date != null)
+ // Write value
+ writeField(out,LITERAL+modifiedDateAttributeName,convertToISO(date));
+ }
+ if (createdDateAttributeName != null)
+ {
+ Date date = document.getCreatedDate();
+ if (date != null)
+ // Write value
+ writeField(out,LITERAL+createdDateAttributeName,convertToISO(date));
+ }
+ if (fileNameAttributeName != null)
+ {
+ String fileName = document.getFileName();
+ if (fileName != null)
+ writeField(out,LITERAL+fileNameAttributeName,fileName);
+ }
+ if (mimeTypeAttributeName != null)
+ {
+ String mimeType = document.getMimeType();
+ if (mimeType != null)
+ writeField(out,LITERAL+mimeTypeAttributeName,mimeType);
+ }
+
// Write the access token information
writeACLs(out,"share",shareAcls,shareDenyAcls);
writeACLs(out,"document",acls,denyAcls);
@@ -935,6 +978,12 @@ public class HttpPoster
}
}
+ protected static String convertToISO(Date date)
+ {
+ java.text.DateFormat df = new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mmZ");
+ return df.format(date);
+ }
+
/** Killable thread that does deletions.
* Java 1.5 stopped permitting thread interruptions to abort socket waits. As a result, it is impossible to get threads to shutdown cleanly that are doing
* such waits. So, the places where this happens are segregated in their own threads so that they can be just abandoned.
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java?rev=1452563&r1=1452562&r2=1452563&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java Mon Mar 4 22:16:08 2013
@@ -83,6 +83,14 @@ public class SolrConfig
public static final String PARAM_STATUSPATH = "Server status handler";
/** Id field */
public static final String PARAM_IDFIELD = "Solr id field name";
+ /** Optional modified date field */
+ public static final String PARAM_MODIFIEDDATEFIELD = "Solr modified date field name";
+ /** Optional created date field */
+ public static final String PARAM_CREATEDDATEFIELD = "Solr created date field name";
+ /** Optional file name field */
+ public static final String PARAM_FILENAMEFIELD = "Solr filename field name";
+ /** Optional mime type field */
+ public static final String PARAM_MIMETYPEFIELD = "Solr mime type field name";
/** Optional basic auth realm */
public static final String PARAM_REALM = "Realm";
/** Optional user ID */
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1452563&r1=1452562&r2=1452563&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java Mon Mar 4 22:16:08 2013
@@ -151,7 +151,23 @@ public class SolrConnector extends org.a
String idAttributeName = params.getParameter(SolrConfig.PARAM_IDFIELD);
if (idAttributeName == null || idAttributeName.length() == 0)
idAttributeName = "id";
-
+
+ String modifiedDateAttributeName = params.getParameter(SolrConfig.PARAM_MODIFIEDDATEFIELD);
+ if (modifiedDateAttributeName == null || modifiedDateAttributeName.length() == 0)
+ modifiedDateAttributeName = null;
+
+ String createdDateAttributeName = params.getParameter(SolrConfig.PARAM_CREATEDDATEFIELD);
+ if (createdDateAttributeName == null || createdDateAttributeName.length() == 0)
+ createdDateAttributeName = null;
+
+ String fileNameAttributeName = params.getParameter(SolrConfig.PARAM_FILENAMEFIELD);
+ if (fileNameAttributeName == null || fileNameAttributeName.length() == 0)
+ fileNameAttributeName = null;
+
+ String mimeTypeAttributeName = params.getParameter(SolrConfig.PARAM_MIMETYPEFIELD);
+ if (mimeTypeAttributeName == null || mimeTypeAttributeName.length() == 0)
+ mimeTypeAttributeName = null;
+
String commits = params.getParameter(SolrConfig.PARAM_COMMITS);
if (commits == null || commits.length() == 0)
commits = "true";
@@ -257,6 +273,8 @@ public class SolrConnector extends org.a
connectTimeout,socketTimeout,
updatePath,removePath,statusPath,realm,userID,password,
allowAttributeName,denyAttributeName,idAttributeName,
+ modifiedDateAttributeName,createdDateAttributeName,
+ fileNameAttributeName,mimeTypeAttributeName,
keystoreManager,maxDocumentLength,commitWithin);
}
@@ -310,6 +328,8 @@ public class SolrConnector extends org.a
zkClientTimeout,zkConnectTimeout,
updatePath,removePath,statusPath,
allowAttributeName,denyAttributeName,idAttributeName,
+ modifiedDateAttributeName,createdDateAttributeName,
+ fileNameAttributeName,mimeTypeAttributeName,
maxDocumentLength,commitWithin);
}
@@ -1032,6 +1052,22 @@ public class SolrConnector extends org.a
if (idField == null)
idField = "id";
+ String modifiedDateField = parameters.getParameter(SolrConfig.PARAM_MODIFIEDDATEFIELD);
+ if (modifiedDateField == null)
+ modifiedDateField = "";
+
+ String createdDateField = parameters.getParameter(SolrConfig.PARAM_CREATEDDATEFIELD);
+ if (createdDateField == null)
+ createdDateField = "";
+
+ String fileNameField = parameters.getParameter(SolrConfig.PARAM_FILENAMEFIELD);
+ if (fileNameField == null)
+ fileNameField = "";
+
+ String mimeTypeField = parameters.getParameter(SolrConfig.PARAM_MIMETYPEFIELD);
+ if (mimeTypeField == null)
+ mimeTypeField = "";
+
String realm = parameters.getParameter(SolrConfig.PARAM_REALM);
if (realm == null)
realm = "";
@@ -1453,13 +1489,41 @@ public class SolrConnector extends org.a
" <input name=\"idfield\" type=\"text\" size=\"32\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(idField)+"\"/>\n"+
" </td>\n"+
" </tr>\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"SolrConnector.ModifiedDateFieldName") + "</nobr></td>\n"+
+" <td class=\"value\">\n"+
+" <input name=\"modifieddatefield\" type=\"text\" size=\"32\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(modifiedDateField)+"\"/>\n"+
+" </td>\n"+
+" </tr>\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"SolrConnector.CreatedDateFieldName") + "</nobr></td>\n"+
+" <td class=\"value\">\n"+
+" <input name=\"createddatefield\" type=\"text\" size=\"32\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(createdDateField)+"\"/>\n"+
+" </td>\n"+
+" </tr>\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"SolrConnector.FileNameFieldName") + "</nobr></td>\n"+
+" <td class=\"value\">\n"+
+" <input name=\"filenamefield\" type=\"text\" size=\"32\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(fileNameField)+"\"/>\n"+
+" </td>\n"+
+" </tr>\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"SolrConnector.MimeTypeFieldName") + "</nobr></td>\n"+
+" <td class=\"value\">\n"+
+" <input name=\"mimetypefield\" type=\"text\" size=\"32\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(mimeTypeField)+"\"/>\n"+
+" </td>\n"+
+" </tr>\n"+
"</table>\n"
);
}
else
{
out.print(
-"<input type=\"hidden\" name=\"idfield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(idField)+"\"/>\n"
+"<input type=\"hidden\" name=\"idfield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(idField)+"\"/>\n"+
+"<input type=\"hidden\" name=\"modifieddatefield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(modifiedDateField)+"\"/>\n"+
+"<input type=\"hidden\" name=\"createddatefield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(createdDateField)+"\"/>\n"+
+"<input type=\"hidden\" name=\"filenamefield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(fileNameField)+"\"/>\n"+
+"<input type=\"hidden\" name=\"mimetypefield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(mimeTypeField)+"\"/>\n"
);
}
@@ -1739,6 +1803,22 @@ public class SolrConnector extends org.a
if (idField != null)
parameters.setParameter(SolrConfig.PARAM_IDFIELD,idField);
+ String modifiedDateField = variableContext.getParameter("modifieddatefield");
+ if (modifiedDateField != null)
+ parameters.setParameter(SolrConfig.PARAM_MODIFIEDDATEFIELD,modifiedDateField);
+
+ String createdDateField = variableContext.getParameter("createddatefield");
+ if (createdDateField != null)
+ parameters.setParameter(SolrConfig.PARAM_CREATEDDATEFIELD,createdDateField);
+
+ String fileNameField = variableContext.getParameter("filenamefield");
+ if (fileNameField != null)
+ parameters.setParameter(SolrConfig.PARAM_FILENAMEFIELD,fileNameField);
+
+ String mimeTypeField = variableContext.getParameter("mimetypefield");
+ if (mimeTypeField != null)
+ parameters.setParameter(SolrConfig.PARAM_MIMETYPEFIELD,mimeTypeField);
+
String realm = variableContext.getParameter("realm");
if (realm != null)
parameters.setParameter(SolrConfig.PARAM_REALM,realm);
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties?rev=1452563&r1=1452562&r2=1452563&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties Mon Mar 4 22:16:08 2013
@@ -52,6 +52,10 @@ SolrConnector.Add=Add
SolrConnector.AddZookeeperHost=Add ZooKeeper host
SolrConnector.Certificate=Certificate:
SolrConnector.IDFieldName=ID field name:
+SolrConnector.ModifiedDateFieldName=Modified date field name:
+SolrConnector.CreatedDateFieldName=Created date field name:
+SolrConnector.FileNameFieldName=File name field name:
+SolrConnector.MimeTypeFieldName=Mime type field name:
SolrConnector.MaximumDocumentLength=Maximum document length:
SolrConnector.IncludedMimeTypes=Included mime types:
SolrConnector.ExcludedMimeTypes=Excluded mime types:
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties?rev=1452563&r1=1452562&r2=1452563&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties Mon Mar 4 22:16:08 2013
@@ -52,6 +52,10 @@ SolrConnector.Add=追å
SolrConnector.AddZookeeperHost=ZooKeeperãã¹ãã追å
SolrConnector.Certificate=証æ証ï¼
SolrConnector.IDFieldName=IDãã£ã¼ã«ãåï¼
+SolrConnector.ModifiedDateFieldName=Modified date field name:
+SolrConnector.CreatedDateFieldName=Created date field name:
+SolrConnector.FileNameFieldName=File name field name:
+SolrConnector.MimeTypeFieldName=Mime type field name:
SolrConnector.MaximumDocumentLength=æ大ã³ã³ãã³ãé·ï¼
SolrConnector.IncludedMimeTypes=å«ãMIMEã¿ã¤ãï¼
SolrConnector.ExcludedMimeTypes=é¤å¤ããMIMEã¿ã¤ãï¼