You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by kl...@apache.org on 2007/09/05 01:05:43 UTC
svn commit: r572831 - in /lucene/solr/trunk: CHANGES.txt
example/solr/conf/solrconfig.xml
src/java/org/apache/solr/update/DirectUpdateHandler2.java
src/test/org/apache/solr/update/AutoCommitTest.java
Author: klaas
Date: Tue Sep 4 16:05:39 2007
New Revision: 572831
URL: http://svn.apache.org/viewvc?rev=572831&view=rev
Log:
SOLR-310: bound pending deletes
Modified:
lucene/solr/trunk/CHANGES.txt
lucene/solr/trunk/example/solr/conf/solrconfig.xml
lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
lucene/solr/trunk/src/test/org/apache/solr/update/AutoCommitTest.java
Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=572831&r1=572830&r2=572831&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Tue Sep 4 16:05:39 2007
@@ -130,7 +130,11 @@
Changes in runtime behavior
Optimizations
- 1. SOLR-276: improve JSON writer speed. (yonik)
+ 1. SOLR-276: improve JSON writer speed. (yonik)
+
+ 2. SOLR-310: bound and reduce memory usage by providing <maxBufferedDeletes> parameter,
+ which flushes deleted without forcing the user to use <commit/> for this purpose.
+ (klaas)
Bug Fixes
1. Make TextField respect sortMissingFirst and sortMissingLast fields.
Modified: lucene/solr/trunk/example/solr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/example/solr/conf/solrconfig.xml?rev=572831&r1=572830&r2=572831&view=diff
==============================================================================
--- lucene/solr/trunk/example/solr/conf/solrconfig.xml (original)
+++ lucene/solr/trunk/example/solr/conf/solrconfig.xml Tue Sep 4 16:05:39 2007
@@ -82,10 +82,19 @@
org.apache.solr.(search|update|request|core|analysis)
-->
- <!-- autocommit pending docs if certain criteria are met
+ <!-- Limit the number of deletions Solr will buffer during doc updating.
+
+ Setting this lower can help bound memory use during indexing.
+ -->
+ <maxPendingDeletes>100000</maxPendingDeletes>
+
+ <!-- Perform a <commit/> automatically under certain conditions:
+
+ maxDocs - number of updates since last commit is greater than this
+ maxTime - oldest uncommited update (in ms) is this long ago
<autoCommit>
<maxDocs>10000</maxDocs>
- <maxTime>1000</maxTime>
+ <maxTime>1000</maxTime>
</autoCommit>
-->
Modified: lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=572831&r1=572830&r2=572831&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java Tue Sep 4 16:05:39 2007
@@ -143,6 +143,7 @@
// The key is the id, the value (Integer) is the number
// of docs to save (delete all except the last "n" added)
protected final Map<String,Integer> pset;
+ protected int maxPendingDeletes = SolrConfig.config.getInt("updateHandler/maxPendingDeletes", -1);
// commonly used constants for the count in the pset
protected final static Integer ZERO = 0;
@@ -274,6 +275,17 @@
numDocsPending.incrementAndGet();
}
}
+ if (maxPendingDeletes > 0 && pset.size() > maxPendingDeletes) {
+ iwCommit.lock();
+ try {
+ // note: this may be entered multiple times since the synchro is
+ // inside the if(), but doDeletions() is a cheap no-op if it has
+ // already executed
+ doDeletions();
+ } finally {
+ iwCommit.unlock();
+ }
+ }
return rc;
}
Modified: lucene/solr/trunk/src/test/org/apache/solr/update/AutoCommitTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/update/AutoCommitTest.java?rev=572831&r1=572830&r2=572831&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/update/AutoCommitTest.java (original)
+++ lucene/solr/trunk/src/test/org/apache/solr/update/AutoCommitTest.java Tue Sep 4 16:05:39 2007
@@ -114,7 +114,7 @@
req.setContentStreams( toContentStreams(
adoc("id", "B15", "subject", "info" ), null ) );
handler.handleRequest( req, rsp );
-
+
assertQ("should find one", req("id:B14") ,"//result[@numFound=1]" );
assertEquals( 2, tracker.autoCommitCount );
assertQ("should find none", req("id:B15") ,"//result[@numFound=0]" );
@@ -176,9 +176,40 @@
Thread.sleep( 1000 );
req.setContentStreams( toContentStreams(
adoc("id", "531", "field_t", "what's inside?", "subject", "info"), null ) );
+ handler.handleRequest( req, rsp );
assertQ("now it should", req("id:500") ,"//result[@numFound=1]" );
assertQ("but not this", req("id:531") ,"//result[@numFound=0]" );
assertEquals( 3, tracker.autoCommitCount );
}
+
+ public void testMaxPending() throws Exception {
+
+ DirectUpdateHandler2 updater = (DirectUpdateHandler2)SolrCore.getSolrCore().getUpdateHandler();
+ updater.maxPendingDeletes = 14;
+
+ XmlUpdateRequestHandler handler = new XmlUpdateRequestHandler();
+ handler.init( null );
+
+ SolrCore core = SolrCore.getSolrCore();
+ MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
+
+ // Add a single document
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
+ for( int i=0; i<14; i++ ) {
+ req.setContentStreams( toContentStreams(
+ adoc("id", "A"+i, "subject", "info" ), null ) );
+ handler.handleRequest( req, rsp );
+ }
+ assertEquals(updater.numDocsPending.get(), 14);
+
+ req.setContentStreams( toContentStreams(
+ adoc("id", "A14", "subject", "info" ), null ) );
+ handler.handleRequest( req, rsp );
+
+ assertEquals(updater.numDocsPending.get(), 0);
+ assertEquals(updater.commitCommands.get(), 0);
+ }
+
}