You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2011/09/06 18:27:21 UTC
svn commit: r1165754 - in /lucene/dev/trunk/solr: ./ contrib/extraction/
contrib/extraction/src/java/org/apache/solr/handler/extraction/
contrib/extraction/src/test/org/apache/solr/handler/extraction/
core/src/java/org/apache/solr/handler/ core/src/tes...
Author: janhoy
Date: Tue Sep 6 16:27:20 2011
New Revision: 1165754
URL: http://svn.apache.org/viewvc?rev=1165754&view=rev
Log:
SOLR-2540: CommitWithin as an Update Request parameter
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/contrib/extraction/CHANGES.txt
lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonLoader.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XMLLoader.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1165754&r1=1165753&r2=1165754&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Tue Sep 6 16:27:20 2011
@@ -340,6 +340,9 @@ Upgrading from Solr 3.3
New Features
----------------------
+* SOLR-2540: CommitWithin as an Update Request parameter
+ You can now specify &commitWithin=N (ms) on the update request (janhoy)
+
* SOLR-2458: post.jar enhanced to handle JSON, CSV and <optimize> (janhoy)
* LUCENE-3234: add a new parameter hl.phraseLimit for FastVectorHighlighter speed up.
Modified: lucene/dev/trunk/solr/contrib/extraction/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/extraction/CHANGES.txt?rev=1165754&r1=1165753&r2=1165754&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/extraction/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/contrib/extraction/CHANGES.txt Tue Sep 6 16:27:20 2011
@@ -30,6 +30,9 @@ $Id$
================== Release 3.4.0-dev ==============
+* SOLR-2540: CommitWithin as an Update Request parameter
+ You can now specify &commitWithin=N (ms) on the update request (janhoy)
+
* SOLR-2743: Remove commons logging. (koji)
================== Release 3.3.0 ==================
Modified: lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java?rev=1165754&r1=1165753&r2=1165754&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java (original)
+++ lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java Tue Sep 6 16:27:20 2011
@@ -97,6 +97,7 @@ public class ExtractingDocumentLoader ex
templateAdd = new AddUpdateCommand(req);
templateAdd.overwrite = params.getBool(UpdateParams.OVERWRITE, true);
+ templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
//this is lightweight
autoDetectParser = new AutoDetectParser(config);
Modified: lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java?rev=1165754&r1=1165753&r2=1165754&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java (original)
+++ lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java Tue Sep 6 16:27:20 2011
@@ -285,6 +285,27 @@ public class ExtractingRequestHandlerTes
assertQ(req("extractedContent:Apache"), "//*[@numFound='1']");
}
+ @Test
+ public void testCommitWithin() throws Exception {
+ ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");
+ assertTrue("handler is null and it shouldn't be", handler != null);
+
+ // Load plain text specifying filename
+ loadLocal("extraction/version_control.txt", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",
+ "fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",
+ "fmap.Author", "extractedAuthor",
+ "literal.id", "one",
+ "fmap.language", "extractedLanguage",
+ "fmap.content", "extractedContent",
+ ExtractingParams.RESOURCE_NAME, "extraction/version_control.txt",
+ "commitWithin", "200"
+ );
+ assertQ(req("id:one"), "//*[@numFound='0']");
+ // TODO: Find better way of testing commitWithin without sleeping?
+ Thread.sleep(1000);
+ assertQ(req("id:one"), "//*[@numFound='1']");
+ }
+
// Note: If you load a plain text file specifying neither MIME type nor filename, extraction will silently fail. This is because Tika's
// automatic MIME type detection will fail, and it will default to using an empty-string-returning default parser
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java?rev=1165754&r1=1165753&r2=1165754&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java Tue Sep 6 16:27:20 2011
@@ -24,7 +24,6 @@ import org.apache.solr.common.SolrInputD
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.ContentStream;
-import static org.apache.solr.handler.XmlUpdateRequestHandler.COMMIT_WITHIN;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;
@@ -97,7 +96,7 @@ public class BinaryUpdateRequestHandler
AddUpdateCommand addCmd = new AddUpdateCommand(req);
addCmd.overwrite = params.getBool(UpdateParams.OVERWRITE, true);
- addCmd.commitWithin = params.getInt(COMMIT_WITHIN, -1);
+ addCmd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
return addCmd;
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java?rev=1165754&r1=1165753&r2=1165754&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java Tue Sep 6 16:27:20 2011
@@ -22,6 +22,7 @@ import org.apache.solr.response.SolrQuer
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.schema.IndexSchema;
@@ -202,7 +203,8 @@ abstract class CSVLoader extends Content
templateAdd = new AddUpdateCommand(req);
templateAdd.overwrite=params.getBool(OVERWRITE,true);
-
+ templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
+
strategy = new CSVStrategy(',', '"', CSVStrategy.COMMENTS_DISABLED, CSVStrategy.ESCAPE_DISABLED, false, false, false, true);
String sep = params.get(SEPARATOR);
if (sep!=null) {
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonLoader.java?rev=1165754&r1=1165753&r2=1165754&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonLoader.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonLoader.java Tue Sep 6 16:27:20 2011
@@ -26,6 +26,7 @@ import org.apache.noggit.JSONParser;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
+import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@@ -54,8 +55,8 @@ class JsonLoader extends ContentStreamLo
this.processor = processor;
this.req = req;
- commitWithin = req.getParams().getInt(XmlUpdateRequestHandler.COMMIT_WITHIN, -1);
- overwrite = req.getParams().getBool(XmlUpdateRequestHandler.OVERWRITE, true);
+ commitWithin = req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1);
+ overwrite = req.getParams().getBool(UpdateParams.OVERWRITE, true);
}
@Override
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XMLLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XMLLoader.java?rev=1165754&r1=1165753&r2=1165754&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XMLLoader.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XMLLoader.java Tue Sep 6 16:27:20 2011
@@ -28,6 +28,7 @@ import org.apache.solr.common.util.Conte
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.commons.io.IOUtils;
@@ -94,6 +95,7 @@ class XMLLoader extends ContentStreamLoa
InstantiationException, IllegalAccessException,
TransformerConfigurationException {
AddUpdateCommand addCmd = null;
+ SolrParams params = req.getParams();
while (true) {
int event = parser.next();
switch (event) {
@@ -108,6 +110,9 @@ class XMLLoader extends ContentStreamLoa
addCmd = new AddUpdateCommand(req);
+ // First look for commitWithin parameter on the request, will be overwritten for individual <add>'s
+ addCmd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
+
for (int i = 0; i < parser.getAttributeCount(); i++) {
String attrName = parser.getAttributeLocalName(i);
String attrVal = parser.getAttributeValue(i);
@@ -121,10 +126,14 @@ class XMLLoader extends ContentStreamLoa
}
} else if ("doc".equals(currTag)) {
- XmlUpdateRequestHandler.log.trace("adding doc...");
- addCmd.clear();
- addCmd.solrDoc = readDoc(parser);
- processor.processAdd(addCmd);
+ if(addCmd != null) {
+ XmlUpdateRequestHandler.log.trace("adding doc...");
+ addCmd.clear();
+ addCmd.solrDoc = readDoc(parser);
+ processor.processAdd(addCmd);
+ } else {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unexpected <doc> tag without an <add> tag surrounding it.");
+ }
} else if (XmlUpdateRequestHandler.COMMIT.equals(currTag) || XmlUpdateRequestHandler.OPTIMIZE.equals(currTag)) {
XmlUpdateRequestHandler.log.trace("parsing " + currTag);
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java?rev=1165754&r1=1165753&r2=1165754&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java Tue Sep 6 16:27:20 2011
@@ -45,6 +45,8 @@ public class XmlUpdateRequestHandler ext
public static final String SOFT_COMMIT = "softCommit";
public static final String OVERWRITE = "overwrite";
+
+ // NOTE: This constant is for use with the <add> XML tag, not the HTTP param with same name
public static final String COMMIT_WITHIN = "commitWithin";
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java?rev=1165754&r1=1165753&r2=1165754&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java Tue Sep 6 16:27:20 2011
@@ -114,6 +114,15 @@ public class TestCSVLoader extends SolrT
}
@Test
+ public void testCommitWithin() throws Exception {
+ makeFile("id\n100\n101\n102");
+ loadLocal("stream.file",filename,"commitWithin","200");
+ assertQ(req("id:[100 TO 110]"),"//*[@numFound='0']");
+ Thread.sleep(1000);
+ assertQ(req("id:[100 TO 110]"),"//*[@numFound='3']");
+ }
+
+ @Test
public void testCommitTrue() throws Exception {
makeFile("id\n100\n101\n102");
loadLocal("stream.file",filename,"commit","true");
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java?rev=1165754&r1=1165753&r2=1165754&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java Tue Sep 6 16:27:20 2011
@@ -24,6 +24,10 @@ import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamReader;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.AddUpdateCommand;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -73,4 +77,29 @@ public class XmlUpdateRequestHandlerTest
assertEquals( 3, out.size() );
assertEquals( "[aaa, bbb, bbb]", out.toString() );
}
+
+ @Test
+ public void testCommitWithin() throws Exception
+ {
+ String xml =
+ "<add>" +
+ " <doc>" +
+ " <field name=\"id\">12345</field>" +
+ " <field name=\"name\">kitten</field>" +
+ " </doc>" +
+ "</add>";
+
+ SolrQueryRequest req = req("commitWithin","100");
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ BufferingRequestProcessor p = new BufferingRequestProcessor(null);
+
+ XMLLoader loader = new XMLLoader(p, inputFactory);
+ loader.load(req, rsp, new ContentStreamBase.StringStream(xml));
+
+ AddUpdateCommand add = p.addCommands.get(0);
+ assertEquals(add.commitWithin, 100);
+
+ req.close();
+ }
+
}
Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java?rev=1165754&r1=1165753&r2=1165754&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java Tue Sep 6 16:27:20 2011
@@ -37,6 +37,9 @@ public interface UpdateParams
/** Commit everything after the command completes */
public static String COMMIT = "commit";
+ /** Commit within a certain time period (in ms) */
+ public static String COMMIT_WITHIN = "commitWithin";
+
/** Optimize the index and commit everything after the command completes */
public static String OPTIMIZE = "optimize";