You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2011/09/15 18:07:32 UTC
svn commit: r1171158 - in /incubator/lcf/trunk: ./
connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/
Author: kwright
Date: Thu Sep 15 16:07:32 2011
New Revision: 1171158
URL: http://svn.apache.org/viewvc?rev=1171158&view=rev
Log:
Fix for CONNECTORS-202. Add ability to set the commit within parameter for Solr.
Modified:
incubator/lcf/trunk/CHANGES.txt
incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Modified: incubator/lcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/CHANGES.txt?rev=1171158&r1=1171157&r2=1171158&view=diff
==============================================================================
--- incubator/lcf/trunk/CHANGES.txt (original)
+++ incubator/lcf/trunk/CHANGES.txt Thu Sep 15 16:07:32 2011
@@ -3,6 +3,9 @@ $Id$
======================= 0.4-dev =====================
+CONNECTORS-202: Add commit-within parameter to Solr output connector.
+(Jan Høydahl, Karl Wright)
+
CONNECTORS-251: Add target "download-dependencies" to build.xml.
(Shinichiro Abe)
Modified: incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1171158&r1=1171157&r2=1171158&view=diff
==============================================================================
--- incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java (original)
+++ incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java Thu Sep 15 16:07:32 2011
@@ -70,9 +70,12 @@ public class HttpPoster
private Long maxDocumentLength;
+ private String commitWithin;
+
private static final String LITERAL = "literal.";
private static final String NOTHING = "__NOTHING__";
private static final String ID_METADATA = "lcf_metadata_id";
+ private static final String COMMITWITHIN_METADATA = "commitWithin";
private int buffersize = 32768; // default buffer size
double sizeCoefficient = 0.0005; // 20 ms additional timeout per 2000 bytes, pulled out of my butt
@@ -131,9 +134,12 @@ public class HttpPoster
String updatePath, String removePath, String statusPath,
String realm, String userID, String password,
String allowAttributeName, String denyAttributeName, String idAttributeName,
- IKeystoreManager keystoreManager, Long maxDocumentLength)
+ IKeystoreManager keystoreManager, Long maxDocumentLength,
+ String commitWithin)
throws ManifoldCFException
{
+ this.commitWithin = commitWithin;
+
this.allowAttributeName = allowAttributeName;
this.denyAttributeName = denyAttributeName;
this.idAttributeName = idAttributeName;
@@ -300,7 +306,7 @@ public class HttpPoster
{
try
{
- IngestThread t = new IngestThread(documentURI,document,arguments,sourceTargets,shareAcls,shareDenyAcls,acls,denyAcls);
+ IngestThread t = new IngestThread(documentURI,document,arguments,sourceTargets,shareAcls,shareDenyAcls,acls,denyAcls,commitWithin);
try
{
t.start();
@@ -1024,7 +1030,8 @@ public class HttpPoster
protected String[] shareDenyAcls;
protected String[] acls;
protected String[] denyAcls;
-
+ protected String commitWithin;
+
protected Long activityStart = null;
protected Long activityBytes = null;
protected String activityCode = null;
@@ -1034,7 +1041,7 @@ public class HttpPoster
protected boolean rval = false;
public IngestThread(String documentURI, RepositoryDocument document, Map arguments, Map sourceTargets,
- String[] shareAcls, String[] shareDenyAcls, String[] acls, String[] denyAcls)
+ String[] shareAcls, String[] shareDenyAcls, String[] acls, String[] denyAcls, String commitWithin)
{
super();
setDaemon(true);
@@ -1046,6 +1053,7 @@ public class HttpPoster
this.acls = acls;
this.denyAcls = denyAcls;
this.sourceTargets = sourceTargets;
+ this.commitWithin = commitWithin;
}
public void run()
@@ -1134,6 +1142,9 @@ public class HttpPoster
}
}
}
+ // Count the commitWithin parameter
+ if (commitWithin != null)
+ totalLength += lengthField(COMMITWITHIN_METADATA,commitWithin);
// Count the binary data
totalLength += lengthPreamble();
totalLength += lengthBoundary("application/octet-stream","myfile","docname");
@@ -1198,6 +1209,10 @@ public class HttpPoster
}
}
}
+
+ // Write the commitWithin parameter
+ if (commitWithin != null)
+ writeField(out,COMMITWITHIN_METADATA,commitWithin);
// Write the content
writePreamble(out);
Modified: incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java?rev=1171158&r1=1171157&r2=1171158&view=diff
==============================================================================
--- incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java (original)
+++ incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java Thu Sep 15 16:07:32 2011
@@ -53,6 +53,8 @@ public class SolrConfig
public static final String PARAM_PASSWORD = "Password";
/** Enable commits */
public static final String PARAM_COMMITS = "Commits";
+ /** Commit within time */
+ public static final String PARAM_COMMITWITHIN = "Commit within";
/** Keystore */
public static final String PARAM_KEYSTORE = "Keystore";
/** Maximum document length */
Modified: incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1171158&r1=1171157&r2=1171158&view=diff
==============================================================================
--- incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java (original)
+++ incubator/lcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java Thu Sep 15 16:07:32 2011
@@ -146,6 +146,10 @@ public class SolrConnector extends org.a
doCommits = commits.equals("true");
+ String commitWithin = params.getParameter(SolrConfig.PARAM_COMMITWITHIN);
+ if (commitWithin == null || commitWithin.length() == 0)
+ commitWithin = null;
+
String docMax = params.getParameter(SolrConfig.PARAM_MAXLENGTH);
if (docMax == null || docMax.length() == 0)
maxDocumentLength = null;
@@ -206,7 +210,7 @@ public class SolrConnector extends org.a
try
{
poster = new HttpPoster(protocol,server,Integer.parseInt(port),webapp,updatePath,removePath,statusPath,realm,userID,password,
- allowAttributeName,denyAttributeName,idAttributeName,keystoreManager,maxDocumentLength);
+ allowAttributeName,denyAttributeName,idAttributeName,keystoreManager,maxDocumentLength,commitWithin);
}
catch (NumberFormatException e)
{
@@ -649,6 +653,12 @@ public class SolrConnector extends org.a
" editconnection.maxdocumentlength.focus();\n"+
" return false;\n"+
" }\n"+
+" if (editconnection.commitwithin.value != \"\" && !isInteger(editconnection.commitwithin.value))\n"+
+" {\n"+
+" alert(\"Commit-within value must be an integer\");\n"+
+" editconnection.commitwithin.focus();\n"+
+" return false;\n"+
+" }\n"+
" return true;\n"+
"}\n"+
"\n"+
@@ -717,6 +727,13 @@ public class SolrConnector extends org.a
" editconnection.maxdocumentlength.focus();\n"+
" return false;\n"+
" }\n"+
+" if (editconnection.commitwithin.value != \"\" && !isInteger(editconnection.commitwithin.value))\n"+
+" {\n"+
+" alert(\"Commit-within value must be an integer\");\n"+
+" SelectTab(\"Commits\");\n"+
+" editconnection.commitwithin.focus();\n"+
+" return false;\n"+
+" }\n"+
" return true;\n"+
"}\n"+
"\n"+
@@ -816,6 +833,10 @@ public class SolrConnector extends org.a
if (commits == null)
commits = "true";
+ String commitWithin = parameters.getParameter(SolrConfig.PARAM_COMMITWITHIN);
+ if (commitWithin == null)
+ commitWithin = "";
+
String solrKeystore = parameters.getParameter(SolrConfig.PARAM_KEYSTORE);
IKeystoreManager localKeystore;
if (solrKeystore == null)
@@ -1050,6 +1071,12 @@ public class SolrConnector extends org.a
" <input name=\"commits\" type=\"checkbox\" value=\"true\""+(commits.equals("true")?" checked=\"yes\"":"")+"/>\n"+
" </td>\n"+
" </tr>\n"+
+" <tr>\n"+
+" <td class=\"description\"><nobr>Commit each document within (ms):</nobr></td>\n"+
+" <td class=\"value\">\n"+
+" <input name=\"commitwithin\" type=\"text\" size=\"16\" value=\""+commitWithin+"\"/>\n"+
+" </td>\n"+
+" </tr>\n"+
"</table>\n"
);
}
@@ -1057,7 +1084,8 @@ public class SolrConnector extends org.a
{
out.print(
"<input type=\"hidden\" name=\"commits_present\" value=\"true\"/>\n"+
-"<input name=\"commits\" type=\"hidden\" value=\""+commits+"\"/>\n"
+"<input name=\"commits\" type=\"hidden\" value=\""+commits+"\"/>\n"+
+"<input name=\"commitwithin\" type=\"hidden\" value=\""+commitWithin+"\"/>\n"
);
}
@@ -1274,6 +1302,10 @@ public class SolrConnector extends org.a
parameters.setParameter(SolrConfig.PARAM_COMMITS,commits);
}
+ String commitWithin = variableContext.getParameter("commitwithin");
+ if (commitWithin != null)
+ parameters.setParameter(SolrConfig.PARAM_COMMITWITHIN,commitWithin);
+
String keystoreValue = variableContext.getParameter("keystoredata");
IKeystoreManager mgr;
if (keystoreValue != null)