You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-dev@lucene.apache.org by Rafał Kuć <ra...@alud.com.pl> on 2009/08/19 19:04:35 UTC
Re: svn commit: r805774 - in /lucene/solr/trunk: ./ src/common/org/apache/solr/common/params/ src/java/org/apache/solr/handler/ src/java/org/apache/solr/update/ src/solrj/org/apache/solr/client/solrj/request/ src/test/org/apache/solr/update/
Hello!
Same here, assertion fails on our hudson.
> On Wed, Aug 19, 2009 at 10:07 PM, Koji
> Sekiguchi<ko...@r.email.ne.jp> wrote:
>> Noble,
>>
>> testExpungeDeletes fails:
> oh . It passed twice in my box.is any one else noticing this failure?
>>
>> <failure message="null" type="junit.framework.AssertionFailedError">
>> junit.framework.AssertionFailedError: null
>> at
>> org.apache.solr.update.DirectUpdateHandlerTest.testExpungeDeletes(DirectUpdateHandlerTest.java:303)
>> </failure>
>>
>> Koji
>>
>> noble@apache.org wrote:
>>>
>>> Author: noble
>>> Date: Wed Aug 19 12:21:22 2009
>>> New Revision: 805774
>>>
>>> URL: http://svn.apache.org/viewvc?rev=805774&view=rev
>>> Log:
>>> SOLR-1275: Add expungeDeletes to DirectUpdateHandler2
>>>
>>> Modified:
>>> lucene/solr/trunk/CHANGES.txt
>>>
>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>
>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>
>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>
>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>
>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>
>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>
>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>
>>> Modified: lucene/solr/trunk/CHANGES.txt
>>> URL:
>>> http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=805774&r1=805773&r2=805774&view=diff
>>>
>>> ==============================================================================
>>> --- lucene/solr/trunk/CHANGES.txt (original)
>>> +++ lucene/solr/trunk/CHANGES.txt Wed Aug 19 12:21:22 2009
>>> @@ -267,7 +267,9 @@
>>> HTMLStripStandardTokenizerFactory deprecated. To strip HTML tags,
>>> HTMLStripCharFilter can be used
>>> with an arbitrary Tokenizer. (koji)
>>> -68. SOLR-1367: Added callback mechanism for converting DocList to
>>> SolrDocumentList in SolrPluginUtils (gsingers) +68. SOLR-1367: Added
>>> callback mechanism for converting DocList to SolrDocumentList in
>>> SolrPluginUtils (gsingers)
>>> +
>>> +68. SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 (noble)
>>> Optimizations
>>>
>>> Modified:
>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>> URL:
>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java?rev=805774&r1=805773&r2=805774&view=diff
>>>
>>> ==============================================================================
>>> ---
>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>> (original)
>>> +++
>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>> Wed Aug 19 12:21:22 2009
>>> @@ -49,4 +49,6 @@
>>> * If optimizing, set the maximum number of segments left in the index
>>> after optimization. 1 is the default (and is equivalent to calling
>>> IndexWriter.optimize() in Lucene).
>>> */
>>> public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments";
>>> +
>>> + public static final String EXPUNGE_DELETES = "expungeDeletes";
>>> }
>>>
>>> Modified:
>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>> URL:
>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java?rev=805774&r1=805773&r2=805774&view=diff
>>>
>>> ==============================================================================
>>> ---
>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>> (original)
>>> +++
>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>> Wed Aug 19 12:21:22 2009
>>> @@ -68,6 +68,7 @@
>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH,
>>> cmd.waitFlush );
>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER,
>>> cmd.waitSearcher );
>>> + cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES,
>>> cmd.expungeDeletes);
>>> cmd.maxOptimizeSegments =
>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
>>> req.getCore().getUpdateHandler().commit( cmd );
>>> @@ -101,6 +102,7 @@
>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH,
>>> cmd.waitFlush );
>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER,
>>> cmd.waitSearcher );
>>> + cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES,
>>> cmd.expungeDeletes); cmd.maxOptimizeSegments =
>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
>>> processor.processCommit( cmd );
>>> return true;
>>>
>>> Modified:
>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>> URL:
>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java?rev=805774&r1=805773&r2=805774&view=diff
>>>
>>> ==============================================================================
>>> --- lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>> (original)
>>> +++ lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java Wed
>>> Aug 19 12:21:22 2009
>>> @@ -155,6 +155,8 @@
>>> sawWaitSearcher = true;
>>> } else if
>>> (UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)) {
>>> cmd.maxOptimizeSegments = Integer.parseInt(attrVal);
>>> + } else if (UpdateParams.EXPUNGE_DELETES.equals(attrName)) {
>>> + cmd.expungeDeletes = StrUtils.parseBoolean(attrVal);
>>> } else {
>>> XmlUpdateRequestHandler.log.warn("unexpected attribute
>>> commit/@" + attrName);
>>> }
>>>
>>> Modified:
>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>> URL:
>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java?rev=805774&r1=805773&r2=805774&view=diff
>>>
>>> ==============================================================================
>>> ---
>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>> (original)
>>> +++
>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>> Wed Aug 19 12:21:22 2009
>>> @@ -23,6 +23,7 @@
>>> public boolean optimize;
>>> public boolean waitFlush;
>>> public boolean waitSearcher=true;
>>> + public boolean expungeDeletes = false;
>>> /**
>>> * During optimize, optimize down to <= this many segments. Must be >=
>>> 1
>>> @@ -39,6 +40,7 @@
>>> return "commit(optimize="+optimize
>>> +",waitFlush="+waitFlush
>>> +",waitSearcher="+waitSearcher
>>> + +",expungeDeletes="+expungeDeletes
>>> +')';
>>> }
>>> }
>>>
>>> Modified:
>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>> URL:
>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java?rev=805774&r1=805773&r2=805774&view=diff
>>>
>>> ==============================================================================
>>> ---
>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>> (original)
>>> +++
>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>> Wed Aug 19 12:21:22 2009
>>> @@ -241,9 +241,10 @@
>>> synchronized (this) {
>>> pset.clear();
>>> closeSearcher(); // flush any deletes
>>> - if (cmd.optimize) {
>>> + if (cmd.optimize || cmd.expungeDeletes) {
>>> openWriter(); // writer needs to be open to optimize
>>> - writer.optimize(cmd.maxOptimizeSegments);
>>> + if(cmd.optimize) writer.optimize(cmd.maxOptimizeSegments);
>>> + if(cmd.expungeDeletes) writer.expungeDeletes(cmd.expungeDeletes);
>>> }
>>> closeWriter();
>>>
>>> Modified:
>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>> URL:
>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=805774&r1=805773&r2=805774&view=diff
>>>
>>> ==============================================================================
>>> ---
>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>> (original)
>>> +++
>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>> Wed Aug 19 12:21:22 2009
>>> @@ -130,6 +130,7 @@
>>> AtomicLong deleteByIdCommandsCumulative= new AtomicLong();
>>> AtomicLong deleteByQueryCommands= new AtomicLong();
>>> AtomicLong deleteByQueryCommandsCumulative= new AtomicLong();
>>> + AtomicLong expungeDeleteCommands = new AtomicLong();
>>> AtomicLong mergeIndexesCommands = new AtomicLong();
>>> AtomicLong commitCommands= new AtomicLong();
>>> AtomicLong optimizeCommands= new AtomicLong();
>>> @@ -382,6 +383,8 @@
>>> if (cmd.optimize) {
>>> optimizeCommands.incrementAndGet();
>>> + } else if (cmd.expungeDeletes) {
>>> + expungeDeleteCommands.incrementAndGet();
>>> } else {
>>> commitCommands.incrementAndGet();
>>> }
>>> @@ -402,6 +405,10 @@
>>> }
>>> closeWriter();
>>> + if (!cmd.optimize && cmd.expungeDeletes) {
>>> + openWriter();
>>> + writer.expungeDeletes();
>>> + }
>>> callPostCommitCallbacks();
>>> if (cmd.optimize) {
>>>
>>> Modified:
>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>> URL:
>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java?rev=805774&r1=805773&r2=805774&view=diff
>>>
>>> ==============================================================================
>>> ---
>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>> (original)
>>> +++
>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>> Wed Aug 19 12:21:22 2009
>>> @@ -153,6 +153,12 @@
>>> return this;
>>> }
>>> + public UpdateRequest setAction(ACTION action, boolean waitFlush,
>>> boolean waitSearcher, int maxSegments , boolean expungeDeletes) {
>>> + setAction(action, waitFlush, waitSearcher,maxSegments) ;
>>> + params.set(UpdateParams.EXPUNGE_DELETES,""+expungeDeletes);
>>> + return this;
>>> + }
>>> +
>>> /**
>>> * @since Solr 1.4
>>> */
>>>
>>> Modified:
>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>> URL:
>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java?rev=805774&r1=805773&r2=805774&view=diff
>>>
>>> ==============================================================================
>>> ---
>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>> (original)
>>> +++
>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>> Wed Aug 19 12:21:22 2009
>>> @@ -17,20 +17,32 @@
>>> package org.apache.solr.update;
>>> +import java.io.IOException;
>>> +import java.util.ArrayList;
>>> import java.util.HashMap;
>>> +import java.util.HashSet;
>>> +import java.util.List;
>>> import java.util.Map;
>>> +import java.util.Set;
>>> import org.apache.lucene.document.Document;
>>> import org.apache.lucene.document.Field;
>>> import org.apache.lucene.document.Field.Index;
>>> import org.apache.lucene.document.Field.Store;
>>> +import org.apache.lucene.index.IndexReader;
>>> +import org.apache.lucene.index.SegmentReader;
>>> +import org.apache.lucene.index.Term;
>>> +import org.apache.lucene.index.TermEnum;
>>> import org.apache.solr.common.SolrException;
>>> import org.apache.solr.common.params.CommonParams;
>>> import org.apache.solr.common.params.MapSolrParams;
>>> import org.apache.solr.core.SolrCore;
>>> import org.apache.solr.request.LocalSolrQueryRequest;
>>> import org.apache.solr.request.SolrQueryRequest;
>>> +import org.apache.solr.search.SolrIndexReader;
>>> +import org.apache.solr.search.SolrIndexSearcher;
>>> import org.apache.solr.util.AbstractSolrTestCase;
>>> +import org.apache.solr.util.RefCounted;
>>> /**
>>> * @@ -247,6 +259,90 @@
>>> );
>>> }
>>> + public void testExpungeDeletes() throws Exception {
>>> + for (int x = 0; x < 3000; x++) {
>>> + addSimpleDoc(x + "");
>>> + }
>>> + SolrCore core = h.getCore();
>>> + UpdateHandler updater = core.getUpdateHandler();
>>> + CommitUpdateCommand cmtCmd = new CommitUpdateCommand(false);
>>> + cmtCmd.waitSearcher = true;
>>> + updater.commit(cmtCmd);
>>> +
>>> + List<String> todelete = new ArrayList<String>();
>>> +
>>> + Set<String> segsdel = new HashSet<String>();
>>> +
>>> + SegmentReader[] sirs = getSegmentReaders(core);
>>> + assertTrue(sirs.length > 6);
>>> + todelete.add(getNthIDTerm(2, sirs[0]));
>>> + segsdel.add(sirs[0].getSegmentName());
>>> + + todelete.add(getNthIDTerm(7, sirs[2]));
>>> + segsdel.add(sirs[2].getSegmentName());
>>> + + todelete.add(getNthIDTerm(4, sirs[5]));
>>> + segsdel.add(sirs[5].getSegmentName());
>>> + + for (String id : todelete) {
>>> + deleteSimpleDoc(id);
>>> + }
>>> + // commit the deletes
>>> + cmtCmd = new CommitUpdateCommand(false);
>>> + cmtCmd.waitSearcher = true;
>>> + updater.commit(cmtCmd);
>>> + + // expunge deletes
>>> + cmtCmd = new CommitUpdateCommand(false);
>>> + cmtCmd.waitSearcher = true;
>>> + cmtCmd.expungeDeletes = true;
>>> + updater.commit(cmtCmd);
>>> + + // we'll have fewer segments
>>> + SegmentReader[] sirs2 = getSegmentReaders(core);
>>> + assertTrue(sirs.length > sirs2.length);
>>> + // check the actual segment names
>>> + for (SegmentReader sr : sirs2) {
>>> + assertTrue(!segsdel.contains(sr.getSegmentName()));
>>> + }
>>> + }
>>> +
>>> + SegmentReader[] getSegmentReaders(SolrCore core) throws IOException {
>>> + RefCounted<SolrIndexSearcher> ref = core.getSearcher(true, true,
>>> null);
>>> + SolrIndexSearcher is = ref.get();
>>> + SegmentReader[] segmentReaders = null;
>>> + try {
>>> + SolrIndexReader reader = is.getReader();
>>> + IndexReader[] subreaders = reader.getSequentialSubReaders();
>>> + segmentReaders = new SegmentReader[subreaders.length];
>>> + for (int x = 0; x < subreaders.length; x++) {
>>> + assert subreaders[x] instanceof SolrIndexReader;
>>> + SolrIndexReader sir = (SolrIndexReader) subreaders[x];
>>> + SegmentReader sr = (SegmentReader) sir.getWrappedReader();
>>> + segmentReaders[x] = sr;
>>> + }
>>> + } finally {
>>> + ref.decref();
>>> + }
>>> + return segmentReaders;
>>> + }
>>> +
>>> + private String getNthIDTerm(int n, IndexReader r) throws IOException {
>>> + TermEnum te = r.terms(new Term("id", ""));
>>> + try {
>>> + int x = 0;
>>> + do {
>>> + if (x >= n) {
>>> + return te.term().text();
>>> + }
>>> + x++;
>>> + } while (te.next());
>>> + } finally {
>>> + te.close();
>>> + }
>>> + return null;
>>> + }
>>> + private void addSimpleDoc(String id) throws Exception {
>>> SolrCore core = h.getCore();
>>>
>>>
>>>
>>>
>>
>>
--
Regards,
Rafał Kuć
Re: svn commit: r805774 - in /lucene/solr/trunk: ./
src/common/org/apache/solr/common/params/ src/java/org/apache/solr/handler/
src/java/org/apache/solr/update/ src/solrj/org/apache/solr/client/solrj/request/
src/test/org/apache/solr/update/
Posted by Jason Rutherglen <ja...@gmail.com>.
Fails consistently with SMS so I'll post a new patch shortly.
On Wed, Aug 19, 2009 at 11:31 AM, Jason
Rutherglen<ja...@gmail.com> wrote:
> It's could be due to CMS so I'll create a config with SMS, run 10
> times and see if it fails.
>
> On Wed, Aug 19, 2009 at 10:59 AM, Koji Sekiguchi<ko...@r.email.ne.jp> wrote:
>> OK, now the problem is gone.
>>
>> Koji
>>
>> Noble Paul നോബിള് नोब्ळ् wrote:
>>>
>>> I'm looking into it.
>>>
>>> I guess I goofed up somewhere
>>>
>>> 2009/8/19 Rafał Kuć <ra...@alud.com.pl>:
>>>
>>>>
>>>> Hello!
>>>>
>>>> Same here, assertion fails on our hudson.
>>>>
>>>>
>>>>
>>>>
>>>>>
>>>>> On Wed, Aug 19, 2009 at 10:07 PM, Koji
>>>>> Sekiguchi<ko...@r.email.ne.jp> wrote:
>>>>>
>>>>>>
>>>>>> Noble,
>>>>>>
>>>>>> testExpungeDeletes fails:
>>>>>>
>>>>>
>>>>> oh . It passed twice in my box.is any one else noticing this failure?
>>>>>
>>>>>>
>>>>>> <failure message="null" type="junit.framework.AssertionFailedError">
>>>>>> junit.framework.AssertionFailedError: null
>>>>>> at
>>>>>>
>>>>>> org.apache.solr.update.DirectUpdateHandlerTest.testExpungeDeletes(DirectUpdateHandlerTest.java:303)
>>>>>> </failure>
>>>>>>
>>>>>> Koji
>>>>>>
>>>>>> noble@apache.org wrote:
>>>>>>
>>>>>>>
>>>>>>> Author: noble
>>>>>>> Date: Wed Aug 19 12:21:22 2009
>>>>>>> New Revision: 805774
>>>>>>>
>>>>>>> URL: http://svn.apache.org/viewvc?rev=805774&view=rev
>>>>>>> Log:
>>>>>>> SOLR-1275: Add expungeDeletes to DirectUpdateHandler2
>>>>>>>
>>>>>>> Modified:
>>>>>>> lucene/solr/trunk/CHANGES.txt
>>>>>>>
>>>>>>>
>>>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>>>>>
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>>>>>
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>>>>>
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>>>>>
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>>>>>
>>>>>>>
>>>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>>>>>
>>>>>>>
>>>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>>>>>
>>>>>>> Modified: lucene/solr/trunk/CHANGES.txt
>>>>>>> URL:
>>>>>>>
>>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>>
>>>>>>>
>>>>>>> ==============================================================================
>>>>>>> --- lucene/solr/trunk/CHANGES.txt (original)
>>>>>>> +++ lucene/solr/trunk/CHANGES.txt Wed Aug 19 12:21:22 2009
>>>>>>> @@ -267,7 +267,9 @@
>>>>>>> HTMLStripStandardTokenizerFactory deprecated. To strip HTML tags,
>>>>>>> HTMLStripCharFilter can be used
>>>>>>> with an arbitrary Tokenizer. (koji)
>>>>>>> -68. SOLR-1367: Added callback mechanism for converting DocList to
>>>>>>> SolrDocumentList in SolrPluginUtils (gsingers) +68. SOLR-1367: Added
>>>>>>> callback mechanism for converting DocList to SolrDocumentList in
>>>>>>> SolrPluginUtils (gsingers)
>>>>>>> +
>>>>>>> +68. SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 (noble)
>>>>>>> Optimizations
>>>>>>>
>>>>>>> Modified:
>>>>>>>
>>>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>>>>> URL:
>>>>>>>
>>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>>
>>>>>>>
>>>>>>> ==============================================================================
>>>>>>> ---
>>>>>>>
>>>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>>>>> (original)
>>>>>>> +++
>>>>>>>
>>>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>>>>> Wed Aug 19 12:21:22 2009
>>>>>>> @@ -49,4 +49,6 @@
>>>>>>> * If optimizing, set the maximum number of segments left in the
>>>>>>> index
>>>>>>> after optimization. 1 is the default (and is equivalent to calling
>>>>>>> IndexWriter.optimize() in Lucene).
>>>>>>> */
>>>>>>> public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments";
>>>>>>> +
>>>>>>> + public static final String EXPUNGE_DELETES = "expungeDeletes";
>>>>>>> }
>>>>>>>
>>>>>>> Modified:
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>>>>> URL:
>>>>>>>
>>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>>
>>>>>>>
>>>>>>> ==============================================================================
>>>>>>> ---
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>>>>> (original)
>>>>>>> +++
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>>>>> Wed Aug 19 12:21:22 2009
>>>>>>> @@ -68,6 +68,7 @@
>>>>>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
>>>>>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH,
>>>>>>> cmd.waitFlush );
>>>>>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER,
>>>>>>> cmd.waitSearcher );
>>>>>>> + cmd.expungeDeletes = params.getBool(
>>>>>>> UpdateParams.EXPUNGE_DELETES,
>>>>>>> cmd.expungeDeletes);
>>>>>>> cmd.maxOptimizeSegments =
>>>>>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS,
>>>>>>> cmd.maxOptimizeSegments);
>>>>>>> req.getCore().getUpdateHandler().commit( cmd );
>>>>>>> @@ -101,6 +102,7 @@
>>>>>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
>>>>>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH,
>>>>>>> cmd.waitFlush );
>>>>>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER,
>>>>>>> cmd.waitSearcher );
>>>>>>> + cmd.expungeDeletes = params.getBool(
>>>>>>> UpdateParams.EXPUNGE_DELETES,
>>>>>>> cmd.expungeDeletes); cmd.maxOptimizeSegments =
>>>>>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS,
>>>>>>> cmd.maxOptimizeSegments);
>>>>>>> processor.processCommit( cmd );
>>>>>>> return true;
>>>>>>>
>>>>>>> Modified:
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>>>>> URL:
>>>>>>>
>>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>>
>>>>>>>
>>>>>>> ==============================================================================
>>>>>>> --- lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>>>>> (original)
>>>>>>> +++ lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>>>>> Wed
>>>>>>> Aug 19 12:21:22 2009
>>>>>>> @@ -155,6 +155,8 @@
>>>>>>> sawWaitSearcher = true;
>>>>>>> } else if
>>>>>>> (UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)) {
>>>>>>> cmd.maxOptimizeSegments = Integer.parseInt(attrVal);
>>>>>>> + } else if
>>>>>>> (UpdateParams.EXPUNGE_DELETES.equals(attrName)) {
>>>>>>> + cmd.expungeDeletes = StrUtils.parseBoolean(attrVal);
>>>>>>> } else {
>>>>>>> XmlUpdateRequestHandler.log.warn("unexpected attribute
>>>>>>> commit/@" + attrName);
>>>>>>> }
>>>>>>>
>>>>>>> Modified:
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>>>>> URL:
>>>>>>>
>>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>>
>>>>>>>
>>>>>>> ==============================================================================
>>>>>>> ---
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>>>>> (original)
>>>>>>> +++
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>>>>> Wed Aug 19 12:21:22 2009
>>>>>>> @@ -23,6 +23,7 @@
>>>>>>> public boolean optimize;
>>>>>>> public boolean waitFlush;
>>>>>>> public boolean waitSearcher=true;
>>>>>>> + public boolean expungeDeletes = false;
>>>>>>> /**
>>>>>>> * During optimize, optimize down to <= this many segments. Must be
>>>>>>> >=
>>>>>>> 1
>>>>>>> @@ -39,6 +40,7 @@
>>>>>>> return "commit(optimize="+optimize
>>>>>>> +",waitFlush="+waitFlush
>>>>>>> +",waitSearcher="+waitSearcher
>>>>>>> + +",expungeDeletes="+expungeDeletes
>>>>>>> +')';
>>>>>>> }
>>>>>>> }
>>>>>>>
>>>>>>> Modified:
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>>>>> URL:
>>>>>>>
>>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>>
>>>>>>>
>>>>>>> ==============================================================================
>>>>>>> ---
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>>>>> (original)
>>>>>>> +++
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>>>>> Wed Aug 19 12:21:22 2009
>>>>>>> @@ -241,9 +241,10 @@
>>>>>>> synchronized (this) {
>>>>>>> pset.clear();
>>>>>>> closeSearcher(); // flush any deletes
>>>>>>> - if (cmd.optimize) {
>>>>>>> + if (cmd.optimize || cmd.expungeDeletes) {
>>>>>>> openWriter(); // writer needs to be open to optimize
>>>>>>> - writer.optimize(cmd.maxOptimizeSegments);
>>>>>>> + if(cmd.optimize) writer.optimize(cmd.maxOptimizeSegments);
>>>>>>> + if(cmd.expungeDeletes)
>>>>>>> writer.expungeDeletes(cmd.expungeDeletes);
>>>>>>> }
>>>>>>> closeWriter();
>>>>>>>
>>>>>>> Modified:
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>>>>> URL:
>>>>>>>
>>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>>
>>>>>>>
>>>>>>> ==============================================================================
>>>>>>> ---
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>>>>> (original)
>>>>>>> +++
>>>>>>>
>>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>>>>> Wed Aug 19 12:21:22 2009
>>>>>>> @@ -130,6 +130,7 @@
>>>>>>> AtomicLong deleteByIdCommandsCumulative= new AtomicLong();
>>>>>>> AtomicLong deleteByQueryCommands= new AtomicLong();
>>>>>>> AtomicLong deleteByQueryCommandsCumulative= new AtomicLong();
>>>>>>> + AtomicLong expungeDeleteCommands = new AtomicLong();
>>>>>>> AtomicLong mergeIndexesCommands = new AtomicLong();
>>>>>>> AtomicLong commitCommands= new AtomicLong();
>>>>>>> AtomicLong optimizeCommands= new AtomicLong();
>>>>>>> @@ -382,6 +383,8 @@
>>>>>>> if (cmd.optimize) {
>>>>>>> optimizeCommands.incrementAndGet();
>>>>>>> + } else if (cmd.expungeDeletes) {
>>>>>>> + expungeDeleteCommands.incrementAndGet();
>>>>>>> } else {
>>>>>>> commitCommands.incrementAndGet();
>>>>>>> }
>>>>>>> @@ -402,6 +405,10 @@
>>>>>>> }
>>>>>>> closeWriter();
>>>>>>> + if (!cmd.optimize && cmd.expungeDeletes) {
>>>>>>> + openWriter();
>>>>>>> + writer.expungeDeletes();
>>>>>>> + }
>>>>>>> callPostCommitCallbacks();
>>>>>>> if (cmd.optimize) {
>>>>>>>
>>>>>>> Modified:
>>>>>>>
>>>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>>>>> URL:
>>>>>>>
>>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>>
>>>>>>>
>>>>>>> ==============================================================================
>>>>>>> ---
>>>>>>>
>>>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>>>>> (original)
>>>>>>> +++
>>>>>>>
>>>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>>>>> Wed Aug 19 12:21:22 2009
>>>>>>> @@ -153,6 +153,12 @@
>>>>>>> return this;
>>>>>>> }
>>>>>>> + public UpdateRequest setAction(ACTION action, boolean waitFlush,
>>>>>>> boolean waitSearcher, int maxSegments , boolean expungeDeletes) {
>>>>>>> + setAction(action, waitFlush, waitSearcher,maxSegments) ;
>>>>>>> + params.set(UpdateParams.EXPUNGE_DELETES,""+expungeDeletes);
>>>>>>> + return this;
>>>>>>> + }
>>>>>>> +
>>>>>>> /**
>>>>>>> * @since Solr 1.4
>>>>>>> */
>>>>>>>
>>>>>>> Modified:
>>>>>>>
>>>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>>>>> URL:
>>>>>>>
>>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>>
>>>>>>>
>>>>>>> ==============================================================================
>>>>>>> ---
>>>>>>>
>>>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>>>>> (original)
>>>>>>> +++
>>>>>>>
>>>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>>>>> Wed Aug 19 12:21:22 2009
>>>>>>> @@ -17,20 +17,32 @@
>>>>>>> package org.apache.solr.update;
>>>>>>> +import java.io.IOException;
>>>>>>> +import java.util.ArrayList;
>>>>>>> import java.util.HashMap;
>>>>>>> +import java.util.HashSet;
>>>>>>> +import java.util.List;
>>>>>>> import java.util.Map;
>>>>>>> +import java.util.Set;
>>>>>>> import org.apache.lucene.document.Document;
>>>>>>> import org.apache.lucene.document.Field;
>>>>>>> import org.apache.lucene.document.Field.Index;
>>>>>>> import org.apache.lucene.document.Field.Store;
>>>>>>> +import org.apache.lucene.index.IndexReader;
>>>>>>> +import org.apache.lucene.index.SegmentReader;
>>>>>>> +import org.apache.lucene.index.Term;
>>>>>>> +import org.apache.lucene.index.TermEnum;
>>>>>>> import org.apache.solr.common.SolrException;
>>>>>>> import org.apache.solr.common.params.CommonParams;
>>>>>>> import org.apache.solr.common.params.MapSolrParams;
>>>>>>> import org.apache.solr.core.SolrCore;
>>>>>>> import org.apache.solr.request.LocalSolrQueryRequest;
>>>>>>> import org.apache.solr.request.SolrQueryRequest;
>>>>>>> +import org.apache.solr.search.SolrIndexReader;
>>>>>>> +import org.apache.solr.search.SolrIndexSearcher;
>>>>>>> import org.apache.solr.util.AbstractSolrTestCase;
>>>>>>> +import org.apache.solr.util.RefCounted;
>>>>>>> /**
>>>>>>> * @@ -247,6 +259,90 @@
>>>>>>> );
>>>>>>> }
>>>>>>> + public void testExpungeDeletes() throws Exception {
>>>>>>> + for (int x = 0; x < 3000; x++) {
>>>>>>> + addSimpleDoc(x + "");
>>>>>>> + }
>>>>>>> + SolrCore core = h.getCore();
>>>>>>> + UpdateHandler updater = core.getUpdateHandler();
>>>>>>> + CommitUpdateCommand cmtCmd = new CommitUpdateCommand(false);
>>>>>>> + cmtCmd.waitSearcher = true;
>>>>>>> + updater.commit(cmtCmd);
>>>>>>> +
>>>>>>> + List<String> todelete = new ArrayList<String>();
>>>>>>> +
>>>>>>> + Set<String> segsdel = new HashSet<String>();
>>>>>>> +
>>>>>>> + SegmentReader[] sirs = getSegmentReaders(core);
>>>>>>> + assertTrue(sirs.length > 6);
>>>>>>> + todelete.add(getNthIDTerm(2, sirs[0]));
>>>>>>> + segsdel.add(sirs[0].getSegmentName());
>>>>>>> + + todelete.add(getNthIDTerm(7, sirs[2]));
>>>>>>> + segsdel.add(sirs[2].getSegmentName());
>>>>>>> + + todelete.add(getNthIDTerm(4, sirs[5]));
>>>>>>> + segsdel.add(sirs[5].getSegmentName());
>>>>>>> + + for (String id : todelete) {
>>>>>>> + deleteSimpleDoc(id);
>>>>>>> + }
>>>>>>> + // commit the deletes
>>>>>>> + cmtCmd = new CommitUpdateCommand(false);
>>>>>>> + cmtCmd.waitSearcher = true;
>>>>>>> + updater.commit(cmtCmd);
>>>>>>> + + // expunge deletes
>>>>>>> + cmtCmd = new CommitUpdateCommand(false);
>>>>>>> + cmtCmd.waitSearcher = true;
>>>>>>> + cmtCmd.expungeDeletes = true;
>>>>>>> + updater.commit(cmtCmd);
>>>>>>> + + // we'll have fewer segments
>>>>>>> + SegmentReader[] sirs2 = getSegmentReaders(core);
>>>>>>> + assertTrue(sirs.length > sirs2.length);
>>>>>>> + // check the actual segment names
>>>>>>> + for (SegmentReader sr : sirs2) {
>>>>>>> + assertTrue(!segsdel.contains(sr.getSegmentName()));
>>>>>>> + }
>>>>>>> + }
>>>>>>> +
>>>>>>> + SegmentReader[] getSegmentReaders(SolrCore core) throws IOException
>>>>>>> {
>>>>>>> + RefCounted<SolrIndexSearcher> ref = core.getSearcher(true, true,
>>>>>>> null);
>>>>>>> + SolrIndexSearcher is = ref.get();
>>>>>>> + SegmentReader[] segmentReaders = null;
>>>>>>> + try {
>>>>>>> + SolrIndexReader reader = is.getReader();
>>>>>>> + IndexReader[] subreaders = reader.getSequentialSubReaders();
>>>>>>> + segmentReaders = new SegmentReader[subreaders.length];
>>>>>>> + for (int x = 0; x < subreaders.length; x++) {
>>>>>>> + assert subreaders[x] instanceof SolrIndexReader;
>>>>>>> + SolrIndexReader sir = (SolrIndexReader) subreaders[x];
>>>>>>> + SegmentReader sr = (SegmentReader) sir.getWrappedReader();
>>>>>>> + segmentReaders[x] = sr;
>>>>>>> + }
>>>>>>> + } finally {
>>>>>>> + ref.decref();
>>>>>>> + }
>>>>>>> + return segmentReaders;
>>>>>>> + }
>>>>>>> +
>>>>>>> + private String getNthIDTerm(int n, IndexReader r) throws
>>>>>>> IOException {
>>>>>>> + TermEnum te = r.terms(new Term("id", ""));
>>>>>>> + try {
>>>>>>> + int x = 0;
>>>>>>> + do {
>>>>>>> + if (x >= n) {
>>>>>>> + return te.term().text();
>>>>>>> + }
>>>>>>> + x++;
>>>>>>> + } while (te.next());
>>>>>>> + } finally {
>>>>>>> + te.close();
>>>>>>> + }
>>>>>>> + return null;
>>>>>>> + }
>>>>>>> + private void addSimpleDoc(String id) throws Exception {
>>>>>>> SolrCore core = h.getCore();
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>
>>>>>>
>>>>
>>>>
>>>>
>>>>
>>>> --
>>>> Regards,
>>>> Rafał Kuć
>>>>
>>>>
>>>>
>>>
>>>
>>>
>>>
>>
>>
>
Re: svn commit: r805774 - in /lucene/solr/trunk: ./
src/common/org/apache/solr/common/params/ src/java/org/apache/solr/handler/
src/java/org/apache/solr/update/ src/solrj/org/apache/solr/client/solrj/request/
src/test/org/apache/solr/update/
Posted by Jason Rutherglen <ja...@gmail.com>.
It's could be due to CMS so I'll create a config with SMS, run 10
times and see if it fails.
On Wed, Aug 19, 2009 at 10:59 AM, Koji Sekiguchi<ko...@r.email.ne.jp> wrote:
> OK, now the problem is gone.
>
> Koji
>
> Noble Paul നോബിള് नोब्ळ् wrote:
>>
>> I'm looking into it.
>>
>> I guess I goofed up somewhere
>>
>> 2009/8/19 Rafał Kuć <ra...@alud.com.pl>:
>>
>>>
>>> Hello!
>>>
>>> Same here, assertion fails on our hudson.
>>>
>>>
>>>
>>>
>>>>
>>>> On Wed, Aug 19, 2009 at 10:07 PM, Koji
>>>> Sekiguchi<ko...@r.email.ne.jp> wrote:
>>>>
>>>>>
>>>>> Noble,
>>>>>
>>>>> testExpungeDeletes fails:
>>>>>
>>>>
>>>> oh . It passed twice in my box.is any one else noticing this failure?
>>>>
>>>>>
>>>>> <failure message="null" type="junit.framework.AssertionFailedError">
>>>>> junit.framework.AssertionFailedError: null
>>>>> at
>>>>>
>>>>> org.apache.solr.update.DirectUpdateHandlerTest.testExpungeDeletes(DirectUpdateHandlerTest.java:303)
>>>>> </failure>
>>>>>
>>>>> Koji
>>>>>
>>>>> noble@apache.org wrote:
>>>>>
>>>>>>
>>>>>> Author: noble
>>>>>> Date: Wed Aug 19 12:21:22 2009
>>>>>> New Revision: 805774
>>>>>>
>>>>>> URL: http://svn.apache.org/viewvc?rev=805774&view=rev
>>>>>> Log:
>>>>>> SOLR-1275: Add expungeDeletes to DirectUpdateHandler2
>>>>>>
>>>>>> Modified:
>>>>>> lucene/solr/trunk/CHANGES.txt
>>>>>>
>>>>>>
>>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>>>>
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>>>>
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>>>>
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>>>>
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>>>>
>>>>>>
>>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>>>>
>>>>>>
>>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>>>>
>>>>>> Modified: lucene/solr/trunk/CHANGES.txt
>>>>>> URL:
>>>>>>
>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>
>>>>>>
>>>>>> ==============================================================================
>>>>>> --- lucene/solr/trunk/CHANGES.txt (original)
>>>>>> +++ lucene/solr/trunk/CHANGES.txt Wed Aug 19 12:21:22 2009
>>>>>> @@ -267,7 +267,9 @@
>>>>>> HTMLStripStandardTokenizerFactory deprecated. To strip HTML tags,
>>>>>> HTMLStripCharFilter can be used
>>>>>> with an arbitrary Tokenizer. (koji)
>>>>>> -68. SOLR-1367: Added callback mechanism for converting DocList to
>>>>>> SolrDocumentList in SolrPluginUtils (gsingers) +68. SOLR-1367: Added
>>>>>> callback mechanism for converting DocList to SolrDocumentList in
>>>>>> SolrPluginUtils (gsingers)
>>>>>> +
>>>>>> +68. SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 (noble)
>>>>>> Optimizations
>>>>>>
>>>>>> Modified:
>>>>>>
>>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>>>> URL:
>>>>>>
>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>
>>>>>>
>>>>>> ==============================================================================
>>>>>> ---
>>>>>>
>>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>>>> (original)
>>>>>> +++
>>>>>>
>>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>>>> Wed Aug 19 12:21:22 2009
>>>>>> @@ -49,4 +49,6 @@
>>>>>> * If optimizing, set the maximum number of segments left in the
>>>>>> index
>>>>>> after optimization. 1 is the default (and is equivalent to calling
>>>>>> IndexWriter.optimize() in Lucene).
>>>>>> */
>>>>>> public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments";
>>>>>> +
>>>>>> + public static final String EXPUNGE_DELETES = "expungeDeletes";
>>>>>> }
>>>>>>
>>>>>> Modified:
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>>>> URL:
>>>>>>
>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>
>>>>>>
>>>>>> ==============================================================================
>>>>>> ---
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>>>> (original)
>>>>>> +++
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>>>> Wed Aug 19 12:21:22 2009
>>>>>> @@ -68,6 +68,7 @@
>>>>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
>>>>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH,
>>>>>> cmd.waitFlush );
>>>>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER,
>>>>>> cmd.waitSearcher );
>>>>>> + cmd.expungeDeletes = params.getBool(
>>>>>> UpdateParams.EXPUNGE_DELETES,
>>>>>> cmd.expungeDeletes);
>>>>>> cmd.maxOptimizeSegments =
>>>>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS,
>>>>>> cmd.maxOptimizeSegments);
>>>>>> req.getCore().getUpdateHandler().commit( cmd );
>>>>>> @@ -101,6 +102,7 @@
>>>>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
>>>>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH,
>>>>>> cmd.waitFlush );
>>>>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER,
>>>>>> cmd.waitSearcher );
>>>>>> + cmd.expungeDeletes = params.getBool(
>>>>>> UpdateParams.EXPUNGE_DELETES,
>>>>>> cmd.expungeDeletes); cmd.maxOptimizeSegments =
>>>>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS,
>>>>>> cmd.maxOptimizeSegments);
>>>>>> processor.processCommit( cmd );
>>>>>> return true;
>>>>>>
>>>>>> Modified:
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>>>> URL:
>>>>>>
>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>
>>>>>>
>>>>>> ==============================================================================
>>>>>> --- lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>>>> (original)
>>>>>> +++ lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>>>> Wed
>>>>>> Aug 19 12:21:22 2009
>>>>>> @@ -155,6 +155,8 @@
>>>>>> sawWaitSearcher = true;
>>>>>> } else if
>>>>>> (UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)) {
>>>>>> cmd.maxOptimizeSegments = Integer.parseInt(attrVal);
>>>>>> + } else if
>>>>>> (UpdateParams.EXPUNGE_DELETES.equals(attrName)) {
>>>>>> + cmd.expungeDeletes = StrUtils.parseBoolean(attrVal);
>>>>>> } else {
>>>>>> XmlUpdateRequestHandler.log.warn("unexpected attribute
>>>>>> commit/@" + attrName);
>>>>>> }
>>>>>>
>>>>>> Modified:
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>>>> URL:
>>>>>>
>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>
>>>>>>
>>>>>> ==============================================================================
>>>>>> ---
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>>>> (original)
>>>>>> +++
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>>>> Wed Aug 19 12:21:22 2009
>>>>>> @@ -23,6 +23,7 @@
>>>>>> public boolean optimize;
>>>>>> public boolean waitFlush;
>>>>>> public boolean waitSearcher=true;
>>>>>> + public boolean expungeDeletes = false;
>>>>>> /**
>>>>>> * During optimize, optimize down to <= this many segments. Must be
>>>>>> >=
>>>>>> 1
>>>>>> @@ -39,6 +40,7 @@
>>>>>> return "commit(optimize="+optimize
>>>>>> +",waitFlush="+waitFlush
>>>>>> +",waitSearcher="+waitSearcher
>>>>>> + +",expungeDeletes="+expungeDeletes
>>>>>> +')';
>>>>>> }
>>>>>> }
>>>>>>
>>>>>> Modified:
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>>>> URL:
>>>>>>
>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>
>>>>>>
>>>>>> ==============================================================================
>>>>>> ---
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>>>> (original)
>>>>>> +++
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>>>> Wed Aug 19 12:21:22 2009
>>>>>> @@ -241,9 +241,10 @@
>>>>>> synchronized (this) {
>>>>>> pset.clear();
>>>>>> closeSearcher(); // flush any deletes
>>>>>> - if (cmd.optimize) {
>>>>>> + if (cmd.optimize || cmd.expungeDeletes) {
>>>>>> openWriter(); // writer needs to be open to optimize
>>>>>> - writer.optimize(cmd.maxOptimizeSegments);
>>>>>> + if(cmd.optimize) writer.optimize(cmd.maxOptimizeSegments);
>>>>>> + if(cmd.expungeDeletes)
>>>>>> writer.expungeDeletes(cmd.expungeDeletes);
>>>>>> }
>>>>>> closeWriter();
>>>>>>
>>>>>> Modified:
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>>>> URL:
>>>>>>
>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>
>>>>>>
>>>>>> ==============================================================================
>>>>>> ---
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>>>> (original)
>>>>>> +++
>>>>>>
>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>>>> Wed Aug 19 12:21:22 2009
>>>>>> @@ -130,6 +130,7 @@
>>>>>> AtomicLong deleteByIdCommandsCumulative= new AtomicLong();
>>>>>> AtomicLong deleteByQueryCommands= new AtomicLong();
>>>>>> AtomicLong deleteByQueryCommandsCumulative= new AtomicLong();
>>>>>> + AtomicLong expungeDeleteCommands = new AtomicLong();
>>>>>> AtomicLong mergeIndexesCommands = new AtomicLong();
>>>>>> AtomicLong commitCommands= new AtomicLong();
>>>>>> AtomicLong optimizeCommands= new AtomicLong();
>>>>>> @@ -382,6 +383,8 @@
>>>>>> if (cmd.optimize) {
>>>>>> optimizeCommands.incrementAndGet();
>>>>>> + } else if (cmd.expungeDeletes) {
>>>>>> + expungeDeleteCommands.incrementAndGet();
>>>>>> } else {
>>>>>> commitCommands.incrementAndGet();
>>>>>> }
>>>>>> @@ -402,6 +405,10 @@
>>>>>> }
>>>>>> closeWriter();
>>>>>> + if (!cmd.optimize && cmd.expungeDeletes) {
>>>>>> + openWriter();
>>>>>> + writer.expungeDeletes();
>>>>>> + }
>>>>>> callPostCommitCallbacks();
>>>>>> if (cmd.optimize) {
>>>>>>
>>>>>> Modified:
>>>>>>
>>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>>>> URL:
>>>>>>
>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>
>>>>>>
>>>>>> ==============================================================================
>>>>>> ---
>>>>>>
>>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>>>> (original)
>>>>>> +++
>>>>>>
>>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>>>> Wed Aug 19 12:21:22 2009
>>>>>> @@ -153,6 +153,12 @@
>>>>>> return this;
>>>>>> }
>>>>>> + public UpdateRequest setAction(ACTION action, boolean waitFlush,
>>>>>> boolean waitSearcher, int maxSegments , boolean expungeDeletes) {
>>>>>> + setAction(action, waitFlush, waitSearcher,maxSegments) ;
>>>>>> + params.set(UpdateParams.EXPUNGE_DELETES,""+expungeDeletes);
>>>>>> + return this;
>>>>>> + }
>>>>>> +
>>>>>> /**
>>>>>> * @since Solr 1.4
>>>>>> */
>>>>>>
>>>>>> Modified:
>>>>>>
>>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>>>> URL:
>>>>>>
>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>>
>>>>>>
>>>>>> ==============================================================================
>>>>>> ---
>>>>>>
>>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>>>> (original)
>>>>>> +++
>>>>>>
>>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>>>> Wed Aug 19 12:21:22 2009
>>>>>> @@ -17,20 +17,32 @@
>>>>>> package org.apache.solr.update;
>>>>>> +import java.io.IOException;
>>>>>> +import java.util.ArrayList;
>>>>>> import java.util.HashMap;
>>>>>> +import java.util.HashSet;
>>>>>> +import java.util.List;
>>>>>> import java.util.Map;
>>>>>> +import java.util.Set;
>>>>>> import org.apache.lucene.document.Document;
>>>>>> import org.apache.lucene.document.Field;
>>>>>> import org.apache.lucene.document.Field.Index;
>>>>>> import org.apache.lucene.document.Field.Store;
>>>>>> +import org.apache.lucene.index.IndexReader;
>>>>>> +import org.apache.lucene.index.SegmentReader;
>>>>>> +import org.apache.lucene.index.Term;
>>>>>> +import org.apache.lucene.index.TermEnum;
>>>>>> import org.apache.solr.common.SolrException;
>>>>>> import org.apache.solr.common.params.CommonParams;
>>>>>> import org.apache.solr.common.params.MapSolrParams;
>>>>>> import org.apache.solr.core.SolrCore;
>>>>>> import org.apache.solr.request.LocalSolrQueryRequest;
>>>>>> import org.apache.solr.request.SolrQueryRequest;
>>>>>> +import org.apache.solr.search.SolrIndexReader;
>>>>>> +import org.apache.solr.search.SolrIndexSearcher;
>>>>>> import org.apache.solr.util.AbstractSolrTestCase;
>>>>>> +import org.apache.solr.util.RefCounted;
>>>>>> /**
>>>>>> * @@ -247,6 +259,90 @@
>>>>>> );
>>>>>> }
>>>>>> + public void testExpungeDeletes() throws Exception {
>>>>>> + for (int x = 0; x < 3000; x++) {
>>>>>> + addSimpleDoc(x + "");
>>>>>> + }
>>>>>> + SolrCore core = h.getCore();
>>>>>> + UpdateHandler updater = core.getUpdateHandler();
>>>>>> + CommitUpdateCommand cmtCmd = new CommitUpdateCommand(false);
>>>>>> + cmtCmd.waitSearcher = true;
>>>>>> + updater.commit(cmtCmd);
>>>>>> +
>>>>>> + List<String> todelete = new ArrayList<String>();
>>>>>> +
>>>>>> + Set<String> segsdel = new HashSet<String>();
>>>>>> +
>>>>>> + SegmentReader[] sirs = getSegmentReaders(core);
>>>>>> + assertTrue(sirs.length > 6);
>>>>>> + todelete.add(getNthIDTerm(2, sirs[0]));
>>>>>> + segsdel.add(sirs[0].getSegmentName());
>>>>>> + + todelete.add(getNthIDTerm(7, sirs[2]));
>>>>>> + segsdel.add(sirs[2].getSegmentName());
>>>>>> + + todelete.add(getNthIDTerm(4, sirs[5]));
>>>>>> + segsdel.add(sirs[5].getSegmentName());
>>>>>> + + for (String id : todelete) {
>>>>>> + deleteSimpleDoc(id);
>>>>>> + }
>>>>>> + // commit the deletes
>>>>>> + cmtCmd = new CommitUpdateCommand(false);
>>>>>> + cmtCmd.waitSearcher = true;
>>>>>> + updater.commit(cmtCmd);
>>>>>> + + // expunge deletes
>>>>>> + cmtCmd = new CommitUpdateCommand(false);
>>>>>> + cmtCmd.waitSearcher = true;
>>>>>> + cmtCmd.expungeDeletes = true;
>>>>>> + updater.commit(cmtCmd);
>>>>>> + + // we'll have fewer segments
>>>>>> + SegmentReader[] sirs2 = getSegmentReaders(core);
>>>>>> + assertTrue(sirs.length > sirs2.length);
>>>>>> + // check the actual segment names
>>>>>> + for (SegmentReader sr : sirs2) {
>>>>>> + assertTrue(!segsdel.contains(sr.getSegmentName()));
>>>>>> + }
>>>>>> + }
>>>>>> +
>>>>>> + SegmentReader[] getSegmentReaders(SolrCore core) throws IOException
>>>>>> {
>>>>>> + RefCounted<SolrIndexSearcher> ref = core.getSearcher(true, true,
>>>>>> null);
>>>>>> + SolrIndexSearcher is = ref.get();
>>>>>> + SegmentReader[] segmentReaders = null;
>>>>>> + try {
>>>>>> + SolrIndexReader reader = is.getReader();
>>>>>> + IndexReader[] subreaders = reader.getSequentialSubReaders();
>>>>>> + segmentReaders = new SegmentReader[subreaders.length];
>>>>>> + for (int x = 0; x < subreaders.length; x++) {
>>>>>> + assert subreaders[x] instanceof SolrIndexReader;
>>>>>> + SolrIndexReader sir = (SolrIndexReader) subreaders[x];
>>>>>> + SegmentReader sr = (SegmentReader) sir.getWrappedReader();
>>>>>> + segmentReaders[x] = sr;
>>>>>> + }
>>>>>> + } finally {
>>>>>> + ref.decref();
>>>>>> + }
>>>>>> + return segmentReaders;
>>>>>> + }
>>>>>> +
>>>>>> + private String getNthIDTerm(int n, IndexReader r) throws
>>>>>> IOException {
>>>>>> + TermEnum te = r.terms(new Term("id", ""));
>>>>>> + try {
>>>>>> + int x = 0;
>>>>>> + do {
>>>>>> + if (x >= n) {
>>>>>> + return te.term().text();
>>>>>> + }
>>>>>> + x++;
>>>>>> + } while (te.next());
>>>>>> + } finally {
>>>>>> + te.close();
>>>>>> + }
>>>>>> + return null;
>>>>>> + }
>>>>>> + private void addSimpleDoc(String id) throws Exception {
>>>>>> SolrCore core = h.getCore();
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>
>>>>>
>>>
>>>
>>>
>>>
>>> --
>>> Regards,
>>> Rafał Kuć
>>>
>>>
>>>
>>
>>
>>
>>
>
>
Re: svn commit: r805774 - in /lucene/solr/trunk: ./ src/common/org/apache/solr/common/params/
src/java/org/apache/solr/handler/ src/java/org/apache/solr/update/ src/solrj/org/apache/solr/client/solrj/request/
src/test/org/apache/solr/update/
Posted by Koji Sekiguchi <ko...@r.email.ne.jp>.
OK, now the problem is gone.
Koji
Noble Paul നോബിള് नोब्ळ् wrote:
> I'm looking into it.
>
> I guess I goofed up somewhere
>
> 2009/8/19 Rafał Kuć <ra...@alud.com.pl>:
>
>> Hello!
>>
>> Same here, assertion fails on our hudson.
>>
>>
>>
>>
>>> On Wed, Aug 19, 2009 at 10:07 PM, Koji
>>> Sekiguchi<ko...@r.email.ne.jp> wrote:
>>>
>>>> Noble,
>>>>
>>>> testExpungeDeletes fails:
>>>>
>>> oh . It passed twice in my box.is any one else noticing this failure?
>>>
>>>> <failure message="null" type="junit.framework.AssertionFailedError">
>>>> junit.framework.AssertionFailedError: null
>>>> at
>>>> org.apache.solr.update.DirectUpdateHandlerTest.testExpungeDeletes(DirectUpdateHandlerTest.java:303)
>>>> </failure>
>>>>
>>>> Koji
>>>>
>>>> noble@apache.org wrote:
>>>>
>>>>> Author: noble
>>>>> Date: Wed Aug 19 12:21:22 2009
>>>>> New Revision: 805774
>>>>>
>>>>> URL: http://svn.apache.org/viewvc?rev=805774&view=rev
>>>>> Log:
>>>>> SOLR-1275: Add expungeDeletes to DirectUpdateHandler2
>>>>>
>>>>> Modified:
>>>>> lucene/solr/trunk/CHANGES.txt
>>>>>
>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>>>
>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>>>
>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>>>
>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>>>
>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>>>
>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>>>
>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>>>
>>>>> Modified: lucene/solr/trunk/CHANGES.txt
>>>>> URL:
>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=805774&r1=805773&r2=805774&view=diff
>>>>>
>>>>> ==============================================================================
>>>>> --- lucene/solr/trunk/CHANGES.txt (original)
>>>>> +++ lucene/solr/trunk/CHANGES.txt Wed Aug 19 12:21:22 2009
>>>>> @@ -267,7 +267,9 @@
>>>>> HTMLStripStandardTokenizerFactory deprecated. To strip HTML tags,
>>>>> HTMLStripCharFilter can be used
>>>>> with an arbitrary Tokenizer. (koji)
>>>>> -68. SOLR-1367: Added callback mechanism for converting DocList to
>>>>> SolrDocumentList in SolrPluginUtils (gsingers) +68. SOLR-1367: Added
>>>>> callback mechanism for converting DocList to SolrDocumentList in
>>>>> SolrPluginUtils (gsingers)
>>>>> +
>>>>> +68. SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 (noble)
>>>>> Optimizations
>>>>>
>>>>> Modified:
>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>>> URL:
>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>
>>>>> ==============================================================================
>>>>> ---
>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>>> (original)
>>>>> +++
>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>>> Wed Aug 19 12:21:22 2009
>>>>> @@ -49,4 +49,6 @@
>>>>> * If optimizing, set the maximum number of segments left in the index
>>>>> after optimization. 1 is the default (and is equivalent to calling
>>>>> IndexWriter.optimize() in Lucene).
>>>>> */
>>>>> public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments";
>>>>> +
>>>>> + public static final String EXPUNGE_DELETES = "expungeDeletes";
>>>>> }
>>>>>
>>>>> Modified:
>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>>> URL:
>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>
>>>>> ==============================================================================
>>>>> ---
>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>>> (original)
>>>>> +++
>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>>> Wed Aug 19 12:21:22 2009
>>>>> @@ -68,6 +68,7 @@
>>>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
>>>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH,
>>>>> cmd.waitFlush );
>>>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER,
>>>>> cmd.waitSearcher );
>>>>> + cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES,
>>>>> cmd.expungeDeletes);
>>>>> cmd.maxOptimizeSegments =
>>>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
>>>>> req.getCore().getUpdateHandler().commit( cmd );
>>>>> @@ -101,6 +102,7 @@
>>>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
>>>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH,
>>>>> cmd.waitFlush );
>>>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER,
>>>>> cmd.waitSearcher );
>>>>> + cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES,
>>>>> cmd.expungeDeletes); cmd.maxOptimizeSegments =
>>>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
>>>>> processor.processCommit( cmd );
>>>>> return true;
>>>>>
>>>>> Modified:
>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>>> URL:
>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>
>>>>> ==============================================================================
>>>>> --- lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>>> (original)
>>>>> +++ lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java Wed
>>>>> Aug 19 12:21:22 2009
>>>>> @@ -155,6 +155,8 @@
>>>>> sawWaitSearcher = true;
>>>>> } else if
>>>>> (UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)) {
>>>>> cmd.maxOptimizeSegments = Integer.parseInt(attrVal);
>>>>> + } else if (UpdateParams.EXPUNGE_DELETES.equals(attrName)) {
>>>>> + cmd.expungeDeletes = StrUtils.parseBoolean(attrVal);
>>>>> } else {
>>>>> XmlUpdateRequestHandler.log.warn("unexpected attribute
>>>>> commit/@" + attrName);
>>>>> }
>>>>>
>>>>> Modified:
>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>>> URL:
>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>
>>>>> ==============================================================================
>>>>> ---
>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>>> (original)
>>>>> +++
>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>>> Wed Aug 19 12:21:22 2009
>>>>> @@ -23,6 +23,7 @@
>>>>> public boolean optimize;
>>>>> public boolean waitFlush;
>>>>> public boolean waitSearcher=true;
>>>>> + public boolean expungeDeletes = false;
>>>>> /**
>>>>> * During optimize, optimize down to <= this many segments. Must be >=
>>>>> 1
>>>>> @@ -39,6 +40,7 @@
>>>>> return "commit(optimize="+optimize
>>>>> +",waitFlush="+waitFlush
>>>>> +",waitSearcher="+waitSearcher
>>>>> + +",expungeDeletes="+expungeDeletes
>>>>> +')';
>>>>> }
>>>>> }
>>>>>
>>>>> Modified:
>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>>> URL:
>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>
>>>>> ==============================================================================
>>>>> ---
>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>>> (original)
>>>>> +++
>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>>> Wed Aug 19 12:21:22 2009
>>>>> @@ -241,9 +241,10 @@
>>>>> synchronized (this) {
>>>>> pset.clear();
>>>>> closeSearcher(); // flush any deletes
>>>>> - if (cmd.optimize) {
>>>>> + if (cmd.optimize || cmd.expungeDeletes) {
>>>>> openWriter(); // writer needs to be open to optimize
>>>>> - writer.optimize(cmd.maxOptimizeSegments);
>>>>> + if(cmd.optimize) writer.optimize(cmd.maxOptimizeSegments);
>>>>> + if(cmd.expungeDeletes) writer.expungeDeletes(cmd.expungeDeletes);
>>>>> }
>>>>> closeWriter();
>>>>>
>>>>> Modified:
>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>>> URL:
>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>
>>>>> ==============================================================================
>>>>> ---
>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>>> (original)
>>>>> +++
>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>>> Wed Aug 19 12:21:22 2009
>>>>> @@ -130,6 +130,7 @@
>>>>> AtomicLong deleteByIdCommandsCumulative= new AtomicLong();
>>>>> AtomicLong deleteByQueryCommands= new AtomicLong();
>>>>> AtomicLong deleteByQueryCommandsCumulative= new AtomicLong();
>>>>> + AtomicLong expungeDeleteCommands = new AtomicLong();
>>>>> AtomicLong mergeIndexesCommands = new AtomicLong();
>>>>> AtomicLong commitCommands= new AtomicLong();
>>>>> AtomicLong optimizeCommands= new AtomicLong();
>>>>> @@ -382,6 +383,8 @@
>>>>> if (cmd.optimize) {
>>>>> optimizeCommands.incrementAndGet();
>>>>> + } else if (cmd.expungeDeletes) {
>>>>> + expungeDeleteCommands.incrementAndGet();
>>>>> } else {
>>>>> commitCommands.incrementAndGet();
>>>>> }
>>>>> @@ -402,6 +405,10 @@
>>>>> }
>>>>> closeWriter();
>>>>> + if (!cmd.optimize && cmd.expungeDeletes) {
>>>>> + openWriter();
>>>>> + writer.expungeDeletes();
>>>>> + }
>>>>> callPostCommitCallbacks();
>>>>> if (cmd.optimize) {
>>>>>
>>>>> Modified:
>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>>> URL:
>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>
>>>>> ==============================================================================
>>>>> ---
>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>>> (original)
>>>>> +++
>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>>> Wed Aug 19 12:21:22 2009
>>>>> @@ -153,6 +153,12 @@
>>>>> return this;
>>>>> }
>>>>> + public UpdateRequest setAction(ACTION action, boolean waitFlush,
>>>>> boolean waitSearcher, int maxSegments , boolean expungeDeletes) {
>>>>> + setAction(action, waitFlush, waitSearcher,maxSegments) ;
>>>>> + params.set(UpdateParams.EXPUNGE_DELETES,""+expungeDeletes);
>>>>> + return this;
>>>>> + }
>>>>> +
>>>>> /**
>>>>> * @since Solr 1.4
>>>>> */
>>>>>
>>>>> Modified:
>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>>> URL:
>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>>
>>>>> ==============================================================================
>>>>> ---
>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>>> (original)
>>>>> +++
>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>>> Wed Aug 19 12:21:22 2009
>>>>> @@ -17,20 +17,32 @@
>>>>> package org.apache.solr.update;
>>>>> +import java.io.IOException;
>>>>> +import java.util.ArrayList;
>>>>> import java.util.HashMap;
>>>>> +import java.util.HashSet;
>>>>> +import java.util.List;
>>>>> import java.util.Map;
>>>>> +import java.util.Set;
>>>>> import org.apache.lucene.document.Document;
>>>>> import org.apache.lucene.document.Field;
>>>>> import org.apache.lucene.document.Field.Index;
>>>>> import org.apache.lucene.document.Field.Store;
>>>>> +import org.apache.lucene.index.IndexReader;
>>>>> +import org.apache.lucene.index.SegmentReader;
>>>>> +import org.apache.lucene.index.Term;
>>>>> +import org.apache.lucene.index.TermEnum;
>>>>> import org.apache.solr.common.SolrException;
>>>>> import org.apache.solr.common.params.CommonParams;
>>>>> import org.apache.solr.common.params.MapSolrParams;
>>>>> import org.apache.solr.core.SolrCore;
>>>>> import org.apache.solr.request.LocalSolrQueryRequest;
>>>>> import org.apache.solr.request.SolrQueryRequest;
>>>>> +import org.apache.solr.search.SolrIndexReader;
>>>>> +import org.apache.solr.search.SolrIndexSearcher;
>>>>> import org.apache.solr.util.AbstractSolrTestCase;
>>>>> +import org.apache.solr.util.RefCounted;
>>>>> /**
>>>>> * @@ -247,6 +259,90 @@
>>>>> );
>>>>> }
>>>>> + public void testExpungeDeletes() throws Exception {
>>>>> + for (int x = 0; x < 3000; x++) {
>>>>> + addSimpleDoc(x + "");
>>>>> + }
>>>>> + SolrCore core = h.getCore();
>>>>> + UpdateHandler updater = core.getUpdateHandler();
>>>>> + CommitUpdateCommand cmtCmd = new CommitUpdateCommand(false);
>>>>> + cmtCmd.waitSearcher = true;
>>>>> + updater.commit(cmtCmd);
>>>>> +
>>>>> + List<String> todelete = new ArrayList<String>();
>>>>> +
>>>>> + Set<String> segsdel = new HashSet<String>();
>>>>> +
>>>>> + SegmentReader[] sirs = getSegmentReaders(core);
>>>>> + assertTrue(sirs.length > 6);
>>>>> + todelete.add(getNthIDTerm(2, sirs[0]));
>>>>> + segsdel.add(sirs[0].getSegmentName());
>>>>> + + todelete.add(getNthIDTerm(7, sirs[2]));
>>>>> + segsdel.add(sirs[2].getSegmentName());
>>>>> + + todelete.add(getNthIDTerm(4, sirs[5]));
>>>>> + segsdel.add(sirs[5].getSegmentName());
>>>>> + + for (String id : todelete) {
>>>>> + deleteSimpleDoc(id);
>>>>> + }
>>>>> + // commit the deletes
>>>>> + cmtCmd = new CommitUpdateCommand(false);
>>>>> + cmtCmd.waitSearcher = true;
>>>>> + updater.commit(cmtCmd);
>>>>> + + // expunge deletes
>>>>> + cmtCmd = new CommitUpdateCommand(false);
>>>>> + cmtCmd.waitSearcher = true;
>>>>> + cmtCmd.expungeDeletes = true;
>>>>> + updater.commit(cmtCmd);
>>>>> + + // we'll have fewer segments
>>>>> + SegmentReader[] sirs2 = getSegmentReaders(core);
>>>>> + assertTrue(sirs.length > sirs2.length);
>>>>> + // check the actual segment names
>>>>> + for (SegmentReader sr : sirs2) {
>>>>> + assertTrue(!segsdel.contains(sr.getSegmentName()));
>>>>> + }
>>>>> + }
>>>>> +
>>>>> + SegmentReader[] getSegmentReaders(SolrCore core) throws IOException {
>>>>> + RefCounted<SolrIndexSearcher> ref = core.getSearcher(true, true,
>>>>> null);
>>>>> + SolrIndexSearcher is = ref.get();
>>>>> + SegmentReader[] segmentReaders = null;
>>>>> + try {
>>>>> + SolrIndexReader reader = is.getReader();
>>>>> + IndexReader[] subreaders = reader.getSequentialSubReaders();
>>>>> + segmentReaders = new SegmentReader[subreaders.length];
>>>>> + for (int x = 0; x < subreaders.length; x++) {
>>>>> + assert subreaders[x] instanceof SolrIndexReader;
>>>>> + SolrIndexReader sir = (SolrIndexReader) subreaders[x];
>>>>> + SegmentReader sr = (SegmentReader) sir.getWrappedReader();
>>>>> + segmentReaders[x] = sr;
>>>>> + }
>>>>> + } finally {
>>>>> + ref.decref();
>>>>> + }
>>>>> + return segmentReaders;
>>>>> + }
>>>>> +
>>>>> + private String getNthIDTerm(int n, IndexReader r) throws IOException {
>>>>> + TermEnum te = r.terms(new Term("id", ""));
>>>>> + try {
>>>>> + int x = 0;
>>>>> + do {
>>>>> + if (x >= n) {
>>>>> + return te.term().text();
>>>>> + }
>>>>> + x++;
>>>>> + } while (te.next());
>>>>> + } finally {
>>>>> + te.close();
>>>>> + }
>>>>> + return null;
>>>>> + }
>>>>> + private void addSimpleDoc(String id) throws Exception {
>>>>> SolrCore core = h.getCore();
>>>>>
>>>>>
>>>>>
>>>>>
>>>>>
>>>>
>>
>>
>>
>>
>> --
>> Regards,
>> Rafał Kuć
>>
>>
>>
>
>
>
>
Re: svn commit: r805774 - in /lucene/solr/trunk: ./
src/common/org/apache/solr/common/params/ src/java/org/apache/solr/handler/
src/java/org/apache/solr/update/ src/solrj/org/apache/solr/client/solrj/request/
src/test/org/apache/solr/update/
Posted by Noble Paul നോബിള് नोब्ळ् <no...@corp.aol.com>.
I'm looking into it.
I guess I goofed up somewhere
2009/8/19 Rafał Kuć <ra...@alud.com.pl>:
> Hello!
>
> Same here, assertion fails on our hudson.
>
>
>
>> On Wed, Aug 19, 2009 at 10:07 PM, Koji
>> Sekiguchi<ko...@r.email.ne.jp> wrote:
>>> Noble,
>>>
>>> testExpungeDeletes fails:
>> oh . It passed twice in my box.is any one else noticing this failure?
>>>
>>> <failure message="null" type="junit.framework.AssertionFailedError">
>>> junit.framework.AssertionFailedError: null
>>> at
>>> org.apache.solr.update.DirectUpdateHandlerTest.testExpungeDeletes(DirectUpdateHandlerTest.java:303)
>>> </failure>
>>>
>>> Koji
>>>
>>> noble@apache.org wrote:
>>>>
>>>> Author: noble
>>>> Date: Wed Aug 19 12:21:22 2009
>>>> New Revision: 805774
>>>>
>>>> URL: http://svn.apache.org/viewvc?rev=805774&view=rev
>>>> Log:
>>>> SOLR-1275: Add expungeDeletes to DirectUpdateHandler2
>>>>
>>>> Modified:
>>>> lucene/solr/trunk/CHANGES.txt
>>>>
>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>>
>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>>
>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>>
>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>>
>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>>
>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>>
>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>>
>>>> Modified: lucene/solr/trunk/CHANGES.txt
>>>> URL:
>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=805774&r1=805773&r2=805774&view=diff
>>>>
>>>> ==============================================================================
>>>> --- lucene/solr/trunk/CHANGES.txt (original)
>>>> +++ lucene/solr/trunk/CHANGES.txt Wed Aug 19 12:21:22 2009
>>>> @@ -267,7 +267,9 @@
>>>> HTMLStripStandardTokenizerFactory deprecated. To strip HTML tags,
>>>> HTMLStripCharFilter can be used
>>>> with an arbitrary Tokenizer. (koji)
>>>> -68. SOLR-1367: Added callback mechanism for converting DocList to
>>>> SolrDocumentList in SolrPluginUtils (gsingers) +68. SOLR-1367: Added
>>>> callback mechanism for converting DocList to SolrDocumentList in
>>>> SolrPluginUtils (gsingers)
>>>> +
>>>> +68. SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 (noble)
>>>> Optimizations
>>>>
>>>> Modified:
>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>> URL:
>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>
>>>> ==============================================================================
>>>> ---
>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>> (original)
>>>> +++
>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
>>>> Wed Aug 19 12:21:22 2009
>>>> @@ -49,4 +49,6 @@
>>>> * If optimizing, set the maximum number of segments left in the index
>>>> after optimization. 1 is the default (and is equivalent to calling
>>>> IndexWriter.optimize() in Lucene).
>>>> */
>>>> public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments";
>>>> +
>>>> + public static final String EXPUNGE_DELETES = "expungeDeletes";
>>>> }
>>>>
>>>> Modified:
>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>> URL:
>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>
>>>> ==============================================================================
>>>> ---
>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>> (original)
>>>> +++
>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
>>>> Wed Aug 19 12:21:22 2009
>>>> @@ -68,6 +68,7 @@
>>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
>>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH,
>>>> cmd.waitFlush );
>>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER,
>>>> cmd.waitSearcher );
>>>> + cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES,
>>>> cmd.expungeDeletes);
>>>> cmd.maxOptimizeSegments =
>>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
>>>> req.getCore().getUpdateHandler().commit( cmd );
>>>> @@ -101,6 +102,7 @@
>>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
>>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH,
>>>> cmd.waitFlush );
>>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER,
>>>> cmd.waitSearcher );
>>>> + cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES,
>>>> cmd.expungeDeletes); cmd.maxOptimizeSegments =
>>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
>>>> processor.processCommit( cmd );
>>>> return true;
>>>>
>>>> Modified:
>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>> URL:
>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>
>>>> ==============================================================================
>>>> --- lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
>>>> (original)
>>>> +++ lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java Wed
>>>> Aug 19 12:21:22 2009
>>>> @@ -155,6 +155,8 @@
>>>> sawWaitSearcher = true;
>>>> } else if
>>>> (UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)) {
>>>> cmd.maxOptimizeSegments = Integer.parseInt(attrVal);
>>>> + } else if (UpdateParams.EXPUNGE_DELETES.equals(attrName)) {
>>>> + cmd.expungeDeletes = StrUtils.parseBoolean(attrVal);
>>>> } else {
>>>> XmlUpdateRequestHandler.log.warn("unexpected attribute
>>>> commit/@" + attrName);
>>>> }
>>>>
>>>> Modified:
>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>> URL:
>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>
>>>> ==============================================================================
>>>> ---
>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>> (original)
>>>> +++
>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
>>>> Wed Aug 19 12:21:22 2009
>>>> @@ -23,6 +23,7 @@
>>>> public boolean optimize;
>>>> public boolean waitFlush;
>>>> public boolean waitSearcher=true;
>>>> + public boolean expungeDeletes = false;
>>>> /**
>>>> * During optimize, optimize down to <= this many segments. Must be >=
>>>> 1
>>>> @@ -39,6 +40,7 @@
>>>> return "commit(optimize="+optimize
>>>> +",waitFlush="+waitFlush
>>>> +",waitSearcher="+waitSearcher
>>>> + +",expungeDeletes="+expungeDeletes
>>>> +')';
>>>> }
>>>> }
>>>>
>>>> Modified:
>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>> URL:
>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>
>>>> ==============================================================================
>>>> ---
>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>> (original)
>>>> +++
>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
>>>> Wed Aug 19 12:21:22 2009
>>>> @@ -241,9 +241,10 @@
>>>> synchronized (this) {
>>>> pset.clear();
>>>> closeSearcher(); // flush any deletes
>>>> - if (cmd.optimize) {
>>>> + if (cmd.optimize || cmd.expungeDeletes) {
>>>> openWriter(); // writer needs to be open to optimize
>>>> - writer.optimize(cmd.maxOptimizeSegments);
>>>> + if(cmd.optimize) writer.optimize(cmd.maxOptimizeSegments);
>>>> + if(cmd.expungeDeletes) writer.expungeDeletes(cmd.expungeDeletes);
>>>> }
>>>> closeWriter();
>>>>
>>>> Modified:
>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>> URL:
>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>
>>>> ==============================================================================
>>>> ---
>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>> (original)
>>>> +++
>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>>> Wed Aug 19 12:21:22 2009
>>>> @@ -130,6 +130,7 @@
>>>> AtomicLong deleteByIdCommandsCumulative= new AtomicLong();
>>>> AtomicLong deleteByQueryCommands= new AtomicLong();
>>>> AtomicLong deleteByQueryCommandsCumulative= new AtomicLong();
>>>> + AtomicLong expungeDeleteCommands = new AtomicLong();
>>>> AtomicLong mergeIndexesCommands = new AtomicLong();
>>>> AtomicLong commitCommands= new AtomicLong();
>>>> AtomicLong optimizeCommands= new AtomicLong();
>>>> @@ -382,6 +383,8 @@
>>>> if (cmd.optimize) {
>>>> optimizeCommands.incrementAndGet();
>>>> + } else if (cmd.expungeDeletes) {
>>>> + expungeDeleteCommands.incrementAndGet();
>>>> } else {
>>>> commitCommands.incrementAndGet();
>>>> }
>>>> @@ -402,6 +405,10 @@
>>>> }
>>>> closeWriter();
>>>> + if (!cmd.optimize && cmd.expungeDeletes) {
>>>> + openWriter();
>>>> + writer.expungeDeletes();
>>>> + }
>>>> callPostCommitCallbacks();
>>>> if (cmd.optimize) {
>>>>
>>>> Modified:
>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>> URL:
>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>
>>>> ==============================================================================
>>>> ---
>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>> (original)
>>>> +++
>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
>>>> Wed Aug 19 12:21:22 2009
>>>> @@ -153,6 +153,12 @@
>>>> return this;
>>>> }
>>>> + public UpdateRequest setAction(ACTION action, boolean waitFlush,
>>>> boolean waitSearcher, int maxSegments , boolean expungeDeletes) {
>>>> + setAction(action, waitFlush, waitSearcher,maxSegments) ;
>>>> + params.set(UpdateParams.EXPUNGE_DELETES,""+expungeDeletes);
>>>> + return this;
>>>> + }
>>>> +
>>>> /**
>>>> * @since Solr 1.4
>>>> */
>>>>
>>>> Modified:
>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>> URL:
>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java?rev=805774&r1=805773&r2=805774&view=diff
>>>>
>>>> ==============================================================================
>>>> ---
>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>> (original)
>>>> +++
>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
>>>> Wed Aug 19 12:21:22 2009
>>>> @@ -17,20 +17,32 @@
>>>> package org.apache.solr.update;
>>>> +import java.io.IOException;
>>>> +import java.util.ArrayList;
>>>> import java.util.HashMap;
>>>> +import java.util.HashSet;
>>>> +import java.util.List;
>>>> import java.util.Map;
>>>> +import java.util.Set;
>>>> import org.apache.lucene.document.Document;
>>>> import org.apache.lucene.document.Field;
>>>> import org.apache.lucene.document.Field.Index;
>>>> import org.apache.lucene.document.Field.Store;
>>>> +import org.apache.lucene.index.IndexReader;
>>>> +import org.apache.lucene.index.SegmentReader;
>>>> +import org.apache.lucene.index.Term;
>>>> +import org.apache.lucene.index.TermEnum;
>>>> import org.apache.solr.common.SolrException;
>>>> import org.apache.solr.common.params.CommonParams;
>>>> import org.apache.solr.common.params.MapSolrParams;
>>>> import org.apache.solr.core.SolrCore;
>>>> import org.apache.solr.request.LocalSolrQueryRequest;
>>>> import org.apache.solr.request.SolrQueryRequest;
>>>> +import org.apache.solr.search.SolrIndexReader;
>>>> +import org.apache.solr.search.SolrIndexSearcher;
>>>> import org.apache.solr.util.AbstractSolrTestCase;
>>>> +import org.apache.solr.util.RefCounted;
>>>> /**
>>>> * @@ -247,6 +259,90 @@
>>>> );
>>>> }
>>>> + public void testExpungeDeletes() throws Exception {
>>>> + for (int x = 0; x < 3000; x++) {
>>>> + addSimpleDoc(x + "");
>>>> + }
>>>> + SolrCore core = h.getCore();
>>>> + UpdateHandler updater = core.getUpdateHandler();
>>>> + CommitUpdateCommand cmtCmd = new CommitUpdateCommand(false);
>>>> + cmtCmd.waitSearcher = true;
>>>> + updater.commit(cmtCmd);
>>>> +
>>>> + List<String> todelete = new ArrayList<String>();
>>>> +
>>>> + Set<String> segsdel = new HashSet<String>();
>>>> +
>>>> + SegmentReader[] sirs = getSegmentReaders(core);
>>>> + assertTrue(sirs.length > 6);
>>>> + todelete.add(getNthIDTerm(2, sirs[0]));
>>>> + segsdel.add(sirs[0].getSegmentName());
>>>> + + todelete.add(getNthIDTerm(7, sirs[2]));
>>>> + segsdel.add(sirs[2].getSegmentName());
>>>> + + todelete.add(getNthIDTerm(4, sirs[5]));
>>>> + segsdel.add(sirs[5].getSegmentName());
>>>> + + for (String id : todelete) {
>>>> + deleteSimpleDoc(id);
>>>> + }
>>>> + // commit the deletes
>>>> + cmtCmd = new CommitUpdateCommand(false);
>>>> + cmtCmd.waitSearcher = true;
>>>> + updater.commit(cmtCmd);
>>>> + + // expunge deletes
>>>> + cmtCmd = new CommitUpdateCommand(false);
>>>> + cmtCmd.waitSearcher = true;
>>>> + cmtCmd.expungeDeletes = true;
>>>> + updater.commit(cmtCmd);
>>>> + + // we'll have fewer segments
>>>> + SegmentReader[] sirs2 = getSegmentReaders(core);
>>>> + assertTrue(sirs.length > sirs2.length);
>>>> + // check the actual segment names
>>>> + for (SegmentReader sr : sirs2) {
>>>> + assertTrue(!segsdel.contains(sr.getSegmentName()));
>>>> + }
>>>> + }
>>>> +
>>>> + SegmentReader[] getSegmentReaders(SolrCore core) throws IOException {
>>>> + RefCounted<SolrIndexSearcher> ref = core.getSearcher(true, true,
>>>> null);
>>>> + SolrIndexSearcher is = ref.get();
>>>> + SegmentReader[] segmentReaders = null;
>>>> + try {
>>>> + SolrIndexReader reader = is.getReader();
>>>> + IndexReader[] subreaders = reader.getSequentialSubReaders();
>>>> + segmentReaders = new SegmentReader[subreaders.length];
>>>> + for (int x = 0; x < subreaders.length; x++) {
>>>> + assert subreaders[x] instanceof SolrIndexReader;
>>>> + SolrIndexReader sir = (SolrIndexReader) subreaders[x];
>>>> + SegmentReader sr = (SegmentReader) sir.getWrappedReader();
>>>> + segmentReaders[x] = sr;
>>>> + }
>>>> + } finally {
>>>> + ref.decref();
>>>> + }
>>>> + return segmentReaders;
>>>> + }
>>>> +
>>>> + private String getNthIDTerm(int n, IndexReader r) throws IOException {
>>>> + TermEnum te = r.terms(new Term("id", ""));
>>>> + try {
>>>> + int x = 0;
>>>> + do {
>>>> + if (x >= n) {
>>>> + return te.term().text();
>>>> + }
>>>> + x++;
>>>> + } while (te.next());
>>>> + } finally {
>>>> + te.close();
>>>> + }
>>>> + return null;
>>>> + }
>>>> + private void addSimpleDoc(String id) throws Exception {
>>>> SolrCore core = h.getCore();
>>>>
>>>>
>>>>
>>>>
>>>
>>>
>
>
>
>
>
>
> --
> Regards,
> Rafał Kuć
>
>
--
-----------------------------------------------------
Noble Paul | Principal Engineer| AOL | http://aol.com