You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/09/09 18:08:00 UTC
svn commit: r1167270 - in /lucene/dev/branches/flexscoring: ./
dev-tools/idea/lucene/contrib/ lucene/ lucene/contrib/
lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/
lucene/contrib/highlighter/src/test/org/apache/lucene/se...
Author: rmuir
Date: Fri Sep 9 16:07:58 2011
New Revision: 1167270
URL: http://svn.apache.org/viewvc?rev=1167270&view=rev
Log:
merge trunk (1166794:1167266)
Added:
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNeverDelete.java
- copied unchanged from r1167266, lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestNeverDelete.java
Modified:
lucene/dev/branches/flexscoring/ (props changed)
lucene/dev/branches/flexscoring/dev-tools/idea/lucene/contrib/ (props changed)
lucene/dev/branches/flexscoring/lucene/ (props changed)
lucene/dev/branches/flexscoring/lucene/contrib/CHANGES.txt
lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FastVectorHighlighter.java
lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java
lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java
lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package.html
lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java
lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldTermStackTest.java
lucene/dev/branches/flexscoring/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java (props changed)
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/MultiFields.java
lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetVectorHighlightTask.java
lucene/dev/branches/flexscoring/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java (props changed)
lucene/dev/branches/flexscoring/solr/ (props changed)
lucene/dev/branches/flexscoring/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/flexscoring/solr/LICENSE.txt (props changed)
lucene/dev/branches/flexscoring/solr/NOTICE.txt (props changed)
lucene/dev/branches/flexscoring/solr/README.txt (props changed)
lucene/dev/branches/flexscoring/solr/build.xml (props changed)
lucene/dev/branches/flexscoring/solr/client/ (props changed)
lucene/dev/branches/flexscoring/solr/common-build.xml (props changed)
lucene/dev/branches/flexscoring/solr/contrib/ (props changed)
lucene/dev/branches/flexscoring/solr/contrib/clustering/src/test-files/ (props changed)
lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler-extras/src/java/ (props changed)
lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/java/ (props changed)
lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/test-files/ (props changed)
lucene/dev/branches/flexscoring/solr/contrib/dataimporthandler/src/test/org/ (props changed)
lucene/dev/branches/flexscoring/solr/contrib/uima/src/java/ (props changed)
lucene/dev/branches/flexscoring/solr/contrib/uima/src/test-files/ (props changed)
lucene/dev/branches/flexscoring/solr/core/ (props changed)
lucene/dev/branches/flexscoring/solr/core/src/java/ (props changed)
lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/update/CommitTracker.java
lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
lucene/dev/branches/flexscoring/solr/core/src/test/ (props changed)
lucene/dev/branches/flexscoring/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java
lucene/dev/branches/flexscoring/solr/dev-tools/ (props changed)
lucene/dev/branches/flexscoring/solr/example/ (props changed)
lucene/dev/branches/flexscoring/solr/lib/ (props changed)
lucene/dev/branches/flexscoring/solr/scripts/ (props changed)
lucene/dev/branches/flexscoring/solr/site/ (props changed)
lucene/dev/branches/flexscoring/solr/site-src/ (props changed)
lucene/dev/branches/flexscoring/solr/solrj/ (props changed)
lucene/dev/branches/flexscoring/solr/solrj/src/java/ (props changed)
lucene/dev/branches/flexscoring/solr/solrj/src/test/org/apache/solr/client/ (props changed)
lucene/dev/branches/flexscoring/solr/solrj/src/test/org/apache/solr/client/solrj/ (props changed)
lucene/dev/branches/flexscoring/solr/solrj/src/test/org/apache/solr/common/ (props changed)
lucene/dev/branches/flexscoring/solr/test-framework/ (props changed)
lucene/dev/branches/flexscoring/solr/testlogging.properties (props changed)
lucene/dev/branches/flexscoring/solr/webapp/ (props changed)
Modified: lucene/dev/branches/flexscoring/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/CHANGES.txt?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/CHANGES.txt Fri Sep 9 16:07:58 2011
@@ -83,6 +83,8 @@ New Features
SimpleBoundaryScanner and BreakIteratorBoundaryScanner, so that FVH's FragmentsBuilder
can find "natural" boundary to make snippets. (Robert Muir, Koji Sekiguchi)
+ * LUCENE-1889: Add MultiTermQuery support for FVH. (Mike Sokolov via Koji Sekiguchi)
+
Bug Fixes
* LUCENE-3417: DictionaryCompoundWordFilter did not properly add tokens from the
Modified: lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FastVectorHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FastVectorHighlighter.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FastVectorHighlighter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FastVectorHighlighter.java Fri Sep 9 16:07:58 2011
@@ -76,8 +76,25 @@ public class FastVectorHighlighter {
* @param query a query
* @return the created {@link FieldQuery} object
*/
- public FieldQuery getFieldQuery( Query query ){
- return new FieldQuery( query, phraseHighlight, fieldMatch );
+ public FieldQuery getFieldQuery( Query query ) {
+ // TODO: should we deprecate this?
+ // because if there is no reader, then we cannot rewrite MTQ.
+ try {
+ return new FieldQuery( query, null, phraseHighlight, fieldMatch );
+ } catch (IOException e) {
+ // should never be thrown when reader is null
+ throw new RuntimeException (e);
+ }
+ }
+
+ /**
+ * create a {@link FieldQuery} object.
+ *
+ * @param query a query
+ * @return the created {@link FieldQuery} object
+ */
+ public FieldQuery getFieldQuery( Query query, IndexReader reader ) throws IOException {
+ return new FieldQuery( query, reader, phraseHighlight, fieldMatch );
}
/**
Modified: lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java Fri Sep 9 16:07:58 2011
@@ -16,6 +16,7 @@ package org.apache.lucene.search.vectorh
* limitations under the License.
*/
+import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -24,10 +25,12 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DisjunctionMaxQuery;
+import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
@@ -51,16 +54,19 @@ public class FieldQuery {
int termOrPhraseNumber; // used for colored tag support
- FieldQuery( Query query, boolean phraseHighlight, boolean fieldMatch ){
+ // The maximum number of different matching terms accumulated from any one MultiTermQuery
+ private static final int MAX_MTQ_TERMS = 1024;
+
+ FieldQuery( Query query, IndexReader reader, boolean phraseHighlight, boolean fieldMatch ) throws IOException {
this.fieldMatch = fieldMatch;
Set<Query> flatQueries = new HashSet<Query>();
- flatten( query, flatQueries );
- saveTerms( flatQueries );
+ flatten( query, reader, flatQueries );
+ saveTerms( flatQueries, reader );
Collection<Query> expandQueries = expand( flatQueries );
for( Query flatQuery : expandQueries ){
QueryPhraseMap rootMap = getRootMap( flatQuery );
- rootMap.add( flatQuery );
+ rootMap.add( flatQuery, reader );
if( !phraseHighlight && flatQuery instanceof PhraseQuery ){
PhraseQuery pq = (PhraseQuery)flatQuery;
if( pq.getTerms().length > 1 ){
@@ -71,24 +77,37 @@ public class FieldQuery {
}
}
- void flatten( Query sourceQuery, Collection<Query> flatQueries ){
+ /** For backwards compatibility you can initialize FieldQuery without
+ * an IndexReader, which is only required to support MultiTermQuery
+ */
+ FieldQuery( Query query, boolean phraseHighlight, boolean fieldMatch ) throws IOException {
+ this (query, null, phraseHighlight, fieldMatch);
+ }
+
+ void flatten( Query sourceQuery, IndexReader reader, Collection<Query> flatQueries ) throws IOException{
if( sourceQuery instanceof BooleanQuery ){
BooleanQuery bq = (BooleanQuery)sourceQuery;
for( BooleanClause clause : bq.getClauses() ){
if( !clause.isProhibited() )
- flatten( clause.getQuery(), flatQueries );
+ flatten( clause.getQuery(), reader, flatQueries );
}
}
else if( sourceQuery instanceof DisjunctionMaxQuery ){
DisjunctionMaxQuery dmq = (DisjunctionMaxQuery)sourceQuery;
for( Query query : dmq ){
- flatten( query, flatQueries );
+ flatten( query, reader, flatQueries );
}
}
else if( sourceQuery instanceof TermQuery ){
if( !flatQueries.contains( sourceQuery ) )
flatQueries.add( sourceQuery );
}
+ else if (sourceQuery instanceof MultiTermQuery && reader != null) {
+ MultiTermQuery copy = (MultiTermQuery) sourceQuery.clone();
+ copy.setRewriteMethod(new MultiTermQuery.TopTermsScoringBooleanQueryRewrite(MAX_MTQ_TERMS));
+ BooleanQuery mtqTerms = (BooleanQuery) copy.rewrite(reader);
+ flatten(mtqTerms, reader, flatQueries);
+ }
else if( sourceQuery instanceof PhraseQuery ){
if( !flatQueries.contains( sourceQuery ) ){
PhraseQuery pq = (PhraseQuery)sourceQuery;
@@ -207,6 +226,9 @@ public class FieldQuery {
Term[] terms = pq.getTerms();
return terms[0].field();
}
+ else if (query instanceof MultiTermQuery) {
+ return ((MultiTermQuery)query).getField();
+ }
else
throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." );
}
@@ -233,7 +255,7 @@ public class FieldQuery {
* - fieldMatch==false
* termSetMap=Map<null,Set<"john","lennon">>
*/
- void saveTerms( Collection<Query> flatQueries ){
+ void saveTerms( Collection<Query> flatQueries, IndexReader reader ) throws IOException{
for( Query query : flatQueries ){
Set<String> termSet = getTermSet( query );
if( query instanceof TermQuery )
@@ -242,6 +264,12 @@ public class FieldQuery {
for( Term term : ((PhraseQuery)query).getTerms() )
termSet.add( term.text() );
}
+ else if (query instanceof MultiTermQuery && reader != null) {
+ BooleanQuery mtqTerms = (BooleanQuery) query.rewrite(reader);
+ for (BooleanClause clause : mtqTerms.getClauses()) {
+ termSet.add (((TermQuery) clause.getQuery()).getTerm().text());
+ }
+ }
else
throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." );
}
@@ -319,7 +347,7 @@ public class FieldQuery {
return map;
}
- void add( Query query ){
+ void add( Query query, IndexReader reader ) throws IOException {
if( query instanceof TermQuery ){
addTerm( ((TermQuery)query).getTerm(), query.getBoost() );
}
Modified: lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java Fri Sep 9 16:07:58 2011
@@ -72,6 +72,10 @@ public class FieldTermStack {
public FieldTermStack( IndexReader reader, int docId, String fieldName, final FieldQuery fieldQuery ) throws IOException {
this.fieldName = fieldName;
+ Set<String> termSet = fieldQuery.getTermSet( fieldName );
+ // just return to make null snippet if un-matched fieldName specified when fieldMatch == true
+ if( termSet == null ) return;
+
TermFreqVector tfv = reader.getTermFreqVector( docId, fieldName );
if( tfv == null ) return; // just return to make null snippets
TermPositionVector tpv = null;
@@ -82,9 +86,6 @@ public class FieldTermStack {
return; // just return to make null snippets
}
- Set<String> termSet = fieldQuery.getTermSet( fieldName );
- // just return to make null snippet if un-matched fieldName specified when fieldMatch == true
- if( termSet == null ) return;
final CharsRef spare = new CharsRef();
for( BytesRef term : tpv.getTerms() ){
if( !termSet.contains( term.utf8ToChars(spare).toString() ) ) continue;
Modified: lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package.html?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package.html (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package.html Fri Sep 9 16:07:58 2011
@@ -24,6 +24,7 @@ This is an another highlighter implement
<li>fast for large docs</li>
<li>support N-gram fields</li>
<li>support phrase-unit highlighting with slops</li>
+<li>support multi-term (includes wildcard, range, regexp, etc) queries</li>
<li>need Java 1.5</li>
<li>highlight fields need to be stored with Positions and Offsets</li>
<li>take into account query boost to score fragments</li>
Modified: lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java Fri Sep 9 16:07:58 2011
@@ -16,19 +16,23 @@ package org.apache.lucene.search.vectorh
* limitations under the License.
*/
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.RegexpQuery;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.vectorhighlight.FieldQuery.QueryPhraseMap;
import org.apache.lucene.search.vectorhighlight.FieldTermStack.TermInfo;
import org.apache.lucene.util.BytesRef;
@@ -48,7 +52,7 @@ public class FieldQueryTest extends Abst
FieldQuery fq = new FieldQuery(booleanQuery, true, true );
Set<Query> flatQueries = new HashSet<Query>();
- fq.flatten(booleanQuery, flatQueries);
+ fq.flatten(booleanQuery, reader, flatQueries);
assertCollectionQueries( flatQueries, tq( "A" ), tq( "B" ), tq( "C" ) );
}
@@ -56,7 +60,7 @@ public class FieldQueryTest extends Abst
Query query = dmq( tq( "A" ), tq( "B" ), pqF( "C", "D" ) );
FieldQuery fq = new FieldQuery( query, true, true );
Set<Query> flatQueries = new HashSet<Query>();
- fq.flatten( query, flatQueries );
+ fq.flatten( query, reader, flatQueries );
assertCollectionQueries( flatQueries, tq( "A" ), tq( "B" ), pqF( "C", "D" ) );
}
@@ -70,7 +74,7 @@ public class FieldQueryTest extends Abst
FieldQuery fq = new FieldQuery(booleanQuery, true, true );
Set<Query> flatQueries = new HashSet<Query>();
- fq.flatten(booleanQuery, flatQueries);
+ fq.flatten(booleanQuery, reader, flatQueries);
assertCollectionQueries( flatQueries, tq( "A" ), pqF( "B", "C" ) );
}
@@ -82,7 +86,7 @@ public class FieldQueryTest extends Abst
FieldQuery fq = new FieldQuery( query, true, true );
Set<Query> flatQueries = new HashSet<Query>();
- fq.flatten( query, flatQueries );
+ fq.flatten( query, reader, flatQueries );
assertCollectionQueries( flatQueries, tq( "AA" ), pqF( "BC", "CD" ), pqF( "EF", "FG", "GH" ) );
}
@@ -90,7 +94,7 @@ public class FieldQueryTest extends Abst
Query query = pqF( "A" );
FieldQuery fq = new FieldQuery( query, true, true );
Set<Query> flatQueries = new HashSet<Query>();
- fq.flatten( query, flatQueries );
+ fq.flatten( query, reader, flatQueries );
assertCollectionQueries( flatQueries, tq( "A" ) );
}
@@ -869,4 +873,36 @@ public class FieldQueryTest extends Abst
phraseCandidate.add( new TermInfo( "c", 4, 5, 6 ) );
assertNull( fq.searchPhrase( F, phraseCandidate ) );
}
+
+ public void testHighlightQuery() throws Exception {
+ makeIndexStrMV();
+ defgMultiTermQueryTest(new WildcardQuery(new Term(F, "d*g")));
+ }
+
+ public void testPrefixQuery() throws Exception {
+ makeIndexStrMV();
+ defgMultiTermQueryTest(new PrefixQuery(new Term(F, "de")));
+ }
+
+ public void testRegexpQuery() throws Exception {
+ makeIndexStrMV();
+ Term term = new Term(F, "d[a-z].g");
+ defgMultiTermQueryTest(new RegexpQuery (term));
+ }
+
+ public void testRangeQuery() throws Exception {
+ makeIndexStrMV();
+ defgMultiTermQueryTest(new TermRangeQuery (F, new BytesRef("d"), new BytesRef("e"), true, true));
+ }
+
+ private void defgMultiTermQueryTest(Query query) throws IOException {
+ FieldQuery fq = new FieldQuery( query, reader, true, true );
+ QueryPhraseMap qpm = fq.getFieldTermMap(F, "defg");
+ assertNotNull (qpm);
+ assertNull (fq.getFieldTermMap(F, "dog"));
+ List<TermInfo> phraseCandidate = new ArrayList<TermInfo>();
+ phraseCandidate.add( new TermInfo( "defg", 0, 12, 0 ) );
+ assertNotNull (fq.searchPhrase(F, phraseCandidate));
+ }
+
}
Modified: lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldTermStackTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldTermStackTest.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldTermStackTest.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldTermStackTest.java Fri Sep 9 16:07:58 2011
@@ -16,8 +16,10 @@ package org.apache.lucene.search.vectorh
* limitations under the License.
*/
+import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.WildcardQuery;
public class FieldTermStackTest extends AbstractTestCase {
@@ -158,4 +160,17 @@ public class FieldTermStackTest extends
assertEquals( "ee(90,92,63)", stack.pop().toString() );
assertEquals( "ed(91,93,64)", stack.pop().toString() );
}
+
+
+ public void testWildcard() throws Exception {
+ makeIndexLongMV();
+ FieldQuery fq = new FieldQuery( new WildcardQuery (new Term(F, "th*e")), reader, true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ assertEquals (4, stack.termList.size());
+ assertEquals ("the(15,18,2)", stack.pop().toString());
+ assertEquals ("these(133,138,20)", stack.pop().toString());
+ assertEquals ("the(153,156,23)", stack.pop().toString());
+ assertEquals ("the(195,198,31)", stack.pop().toString());
+ }
+
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/CheckIndex.java Fri Sep 9 16:07:58 2011
@@ -886,87 +886,99 @@ public class CheckIndex {
}
final Terms fieldTerms = fields.terms(field);
- if (fieldTerms instanceof BlockTreeTermsReader.FieldReader) {
- final BlockTreeTermsReader.Stats stats = ((BlockTreeTermsReader.FieldReader) fieldTerms).computeStats();
- assert stats != null;
- if (status.blockTreeStats == null) {
- status.blockTreeStats = new HashMap<String,BlockTreeTermsReader.Stats>();
+ if (fieldTerms == null) {
+ // Unusual: the FieldsEnum returned a field but
+ // the Terms for that field is null; this should
+ // only happen if it's a ghost field (field with
+ // no terms, eg there used to be terms but all
+ // docs got deleted and then merged away):
+ // make sure TermsEnum is empty:
+ if (fieldsEnum.terms().next() != null) {
+ throw new RuntimeException("Fields.terms(field=" + field + ") returned null yet the field appears to have terms");
+ }
+ } else {
+ if (fieldTerms instanceof BlockTreeTermsReader.FieldReader) {
+ final BlockTreeTermsReader.Stats stats = ((BlockTreeTermsReader.FieldReader) fieldTerms).computeStats();
+ assert stats != null;
+ if (status.blockTreeStats == null) {
+ status.blockTreeStats = new HashMap<String,BlockTreeTermsReader.Stats>();
+ }
+ status.blockTreeStats.put(field, stats);
}
- status.blockTreeStats.put(field, stats);
- }
- if (sumTotalTermFreq != 0) {
- final long v = fields.terms(field).getSumTotalTermFreq();
- if (v != -1 && sumTotalTermFreq != v) {
- throw new RuntimeException("sumTotalTermFreq for field " + field + "=" + v + " != recomputed sumTotalTermFreq=" + sumTotalTermFreq);
+ if (sumTotalTermFreq != 0) {
+ final long v = fields.terms(field).getSumTotalTermFreq();
+ if (v != -1 && sumTotalTermFreq != v) {
+ throw new RuntimeException("sumTotalTermFreq for field " + field + "=" + v + " != recomputed sumTotalTermFreq=" + sumTotalTermFreq);
+ }
}
- }
- if (sumDocFreq != 0) {
- final long v = fields.terms(field).getSumDocFreq();
- if (v != -1 && sumDocFreq != v) {
- throw new RuntimeException("sumDocFreq for field " + field + "=" + v + " != recomputed sumDocFreq=" + sumDocFreq);
- }
- }
-
- // Test seek to last term:
- if (lastTerm != null) {
- if (terms.seekCeil(lastTerm) != TermsEnum.SeekStatus.FOUND) {
- throw new RuntimeException("seek to last term " + lastTerm + " failed");
+ if (sumDocFreq != 0) {
+ final long v = fields.terms(field).getSumDocFreq();
+ if (v != -1 && sumDocFreq != v) {
+ throw new RuntimeException("sumDocFreq for field " + field + "=" + v + " != recomputed sumDocFreq=" + sumDocFreq);
+ }
}
- is.search(new TermQuery(new Term(field, lastTerm)), 1);
- }
+ // Test seek to last term:
+ if (lastTerm != null) {
+ if (terms.seekCeil(lastTerm) != TermsEnum.SeekStatus.FOUND) {
+ throw new RuntimeException("seek to last term " + lastTerm + " failed");
+ }
- // Test seeking by ord
- if (hasOrd && status.termCount-termCountStart > 0) {
- long termCount;
- try {
- termCount = fields.terms(field).getUniqueTermCount();
- } catch (UnsupportedOperationException uoe) {
- termCount = -1;
+ is.search(new TermQuery(new Term(field, lastTerm)), 1);
}
- if (termCount != -1 && termCount != status.termCount - termCountStart) {
- throw new RuntimeException("termCount mismatch " + termCount + " vs " + (status.termCount - termCountStart));
- }
+ // Test seeking by ord
+ if (hasOrd && status.termCount-termCountStart > 0) {
+ long termCount;
+ try {
+ termCount = fields.terms(field).getUniqueTermCount();
+ } catch (UnsupportedOperationException uoe) {
+ termCount = -1;
+ }
- int seekCount = (int) Math.min(10000L, termCount);
- if (seekCount > 0) {
- BytesRef[] seekTerms = new BytesRef[seekCount];
-
- // Seek by ord
- for(int i=seekCount-1;i>=0;i--) {
- long ord = i*(termCount/seekCount);
- terms.seekExact(ord);
- seekTerms[i] = new BytesRef(terms.term());
+ if (termCount != -1 && termCount != status.termCount - termCountStart) {
+ throw new RuntimeException("termCount mismatch " + termCount + " vs " + (status.termCount - termCountStart));
}
- // Seek by term
- long totDocCount = 0;
- for(int i=seekCount-1;i>=0;i--) {
- if (terms.seekCeil(seekTerms[i]) != TermsEnum.SeekStatus.FOUND) {
- throw new RuntimeException("seek to existing term " + seekTerms[i] + " failed");
+ int seekCount = (int) Math.min(10000L, termCount);
+ if (seekCount > 0) {
+ BytesRef[] seekTerms = new BytesRef[seekCount];
+
+ // Seek by ord
+ for(int i=seekCount-1;i>=0;i--) {
+ long ord = i*(termCount/seekCount);
+ terms.seekExact(ord);
+ seekTerms[i] = new BytesRef(terms.term());
}
+
+ // Seek by term
+ long totDocCount = 0;
+ for(int i=seekCount-1;i>=0;i--) {
+ if (terms.seekCeil(seekTerms[i]) != TermsEnum.SeekStatus.FOUND) {
+ throw new RuntimeException("seek to existing term " + seekTerms[i] + " failed");
+ }
- docs = terms.docs(liveDocs, docs);
- if (docs == null) {
- throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]);
+ docs = terms.docs(liveDocs, docs);
+ if (docs == null) {
+ throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]);
+ }
+
+ while(docs.nextDoc() != DocsEnum.NO_MORE_DOCS) {
+ totDocCount++;
+ }
}
- while(docs.nextDoc() != DocsEnum.NO_MORE_DOCS) {
- totDocCount++;
+ // TermQuery
+ long totDocCount2 = 0;
+ for(int i=0;i<seekCount;i++) {
+ totDocCount2 += is.search(new TermQuery(new Term(field, seekTerms[i])), 1).totalHits;
}
- }
- // TermQuery
- long totDocCount2 = 0;
- for(int i=0;i<seekCount;i++) {
- totDocCount2 += is.search(new TermQuery(new Term(field, seekTerms[i])), 1).totalHits;
- }
-
- if (totDocCount != totDocCount2) {
- throw new RuntimeException("search to seek terms produced wrong number of hits: " + totDocCount + " vs " + totDocCount2);
+ if (totDocCount != totDocCount2) {
+ throw new RuntimeException("search to seek terms produced wrong number of hits: " + totDocCount + " vs " + totDocCount2);
+ }
}
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Fri Sep 9 16:07:58 2011
@@ -723,7 +723,7 @@ class DirectoryReader extends IndexReade
// KeepOnlyLastCommitDeleter:
IndexFileDeleter deleter = new IndexFileDeleter(directory,
deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy,
- segmentInfos, null, codecs);
+ segmentInfos, null, codecs, null);
segmentInfos.updateGeneration(deleter.getLastSegmentInfos());
segmentInfos.changed();
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java Fri Sep 9 16:07:58 2011
@@ -108,6 +108,9 @@ final class IndexFileDeleter {
* infoStream != null */
public static boolean VERBOSE_REF_COUNTS = false;
+ // Used only for assert
+ private final IndexWriter writer;
+
void setInfoStream(PrintStream infoStream) {
this.infoStream = infoStream;
if (infoStream != null) {
@@ -121,6 +124,11 @@ final class IndexFileDeleter {
private final FilenameFilter indexFilenameFilter;
+ // called only from assert
+ private boolean locked() {
+ return writer == null || Thread.holdsLock(writer);
+ }
+
/**
* Initialize the deleter: find all previous commits in
* the Directory, incref the files they reference, call
@@ -129,8 +137,10 @@ final class IndexFileDeleter {
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, CodecProvider codecs) throws CorruptIndexException, IOException {
+ public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos,
+ PrintStream infoStream, CodecProvider codecs, IndexWriter writer) throws CorruptIndexException, IOException {
this.infoStream = infoStream;
+ this.writer = writer;
final String currentSegmentsFile = segmentInfos.getCurrentSegmentFileName();
@@ -347,6 +357,8 @@ final class IndexFileDeleter {
* that segment.
*/
public void refresh(String segmentName) throws IOException {
+ assert locked();
+
String[] files = directory.listAll();
String segmentPrefix1;
String segmentPrefix2;
@@ -377,12 +389,14 @@ final class IndexFileDeleter {
// Set to null so that we regenerate the list of pending
// files; else we can accumulate same file more than
// once
+ assert locked();
deletable = null;
refresh(null);
}
public void close() throws IOException {
// DecRef old files from the last checkpoint, if any:
+ assert locked();
int size = lastFiles.size();
if (size > 0) {
for(int i=0;i<size;i++) {
@@ -404,6 +418,7 @@ final class IndexFileDeleter {
* unused commits again.
*/
void revisitPolicy() throws IOException {
+ assert locked();
if (infoStream != null) {
message("now revisitPolicy");
}
@@ -415,6 +430,7 @@ final class IndexFileDeleter {
}
public void deletePendingFiles() throws IOException {
+ assert locked();
if (deletable != null) {
List<String> oldDeletable = deletable;
deletable = null;
@@ -449,6 +465,7 @@ final class IndexFileDeleter {
* removed, we decref their files as well.
*/
public void checkpoint(SegmentInfos segmentInfos, boolean isCommit) throws IOException {
+ assert locked();
if (infoStream != null) {
message("now checkpoint \"" + segmentInfos.toString(directory) + "\" [" + segmentInfos.size() + " segments " + "; isCommit = " + isCommit + "]");
@@ -483,6 +500,7 @@ final class IndexFileDeleter {
}
void incRef(SegmentInfos segmentInfos, boolean isCommit) throws IOException {
+ assert locked();
// If this is a commit point, also incRef the
// segments_N file:
for( final String fileName: segmentInfos.files(directory, isCommit) ) {
@@ -491,12 +509,14 @@ final class IndexFileDeleter {
}
void incRef(Collection<String> files) throws IOException {
+ assert locked();
for(final String file : files) {
incRef(file);
}
}
void incRef(String fileName) throws IOException {
+ assert locked();
RefCount rc = getRefCount(fileName);
if (infoStream != null && VERBOSE_REF_COUNTS) {
message(" IncRef \"" + fileName + "\": pre-incr count is " + rc.count);
@@ -505,12 +525,14 @@ final class IndexFileDeleter {
}
void decRef(Collection<String> files) throws IOException {
+ assert locked();
for(final String file : files) {
decRef(file);
}
}
void decRef(String fileName) throws IOException {
+ assert locked();
RefCount rc = getRefCount(fileName);
if (infoStream != null && VERBOSE_REF_COUNTS) {
message(" DecRef \"" + fileName + "\": pre-decr count is " + rc.count);
@@ -524,12 +546,14 @@ final class IndexFileDeleter {
}
void decRef(SegmentInfos segmentInfos) throws IOException {
+ assert locked();
for (final String file : segmentInfos.files(directory, false)) {
decRef(file);
}
}
public boolean exists(String fileName) {
+ assert locked();
if (!refCounts.containsKey(fileName)) {
return false;
} else {
@@ -538,6 +562,7 @@ final class IndexFileDeleter {
}
private RefCount getRefCount(String fileName) {
+ assert locked();
RefCount rc;
if (!refCounts.containsKey(fileName)) {
rc = new RefCount(fileName);
@@ -549,6 +574,7 @@ final class IndexFileDeleter {
}
void deleteFiles(List<String> files) throws IOException {
+ assert locked();
for(final String file: files) {
deleteFile(file);
}
@@ -557,6 +583,7 @@ final class IndexFileDeleter {
/** Deletes the specified files, but only if they are new
* (have not yet been incref'd). */
void deleteNewFiles(Collection<String> files) throws IOException {
+ assert locked();
for (final String fileName: files) {
if (!refCounts.containsKey(fileName)) {
if (infoStream != null) {
@@ -569,6 +596,7 @@ final class IndexFileDeleter {
void deleteFile(String fileName)
throws IOException {
+ assert locked();
try {
if (infoStream != null) {
message("delete \"" + fileName + "\"");
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/IndexWriter.java Fri Sep 9 16:07:58 2011
@@ -910,9 +910,12 @@ public class IndexWriter implements Clos
// Default deleter (for backwards compatibility) is
// KeepOnlyLastCommitDeleter:
- deleter = new IndexFileDeleter(directory,
- conf.getIndexDeletionPolicy(),
- segmentInfos, infoStream, codecs);
+ synchronized(this) {
+ deleter = new IndexFileDeleter(directory,
+ conf.getIndexDeletionPolicy(),
+ segmentInfos, infoStream,
+ codecs, this);
+ }
if (deleter.startingCommitDeleted) {
// Deletion policy deleted the "head" commit point.
@@ -2578,7 +2581,9 @@ public class IndexWriter implements Clos
// delete new non cfs files directly: they were never
// registered with IFD
- deleter.deleteNewFiles(info.files());
+ synchronized(this) {
+ deleter.deleteNewFiles(info.files());
+ }
info.setUseCompoundFile(true);
}
@@ -2804,7 +2809,9 @@ public class IndexWriter implements Clos
success = true;
} finally {
if (!success) {
- deleter.decRef(toCommit);
+ synchronized (this) {
+ deleter.decRef(toCommit);
+ }
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/MultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/MultiFields.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/MultiFields.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/MultiFields.java Fri Sep 9 16:07:58 2011
@@ -51,7 +51,8 @@ public final class MultiFields extends F
/** Returns a single {@link Fields} instance for this
* reader, merging fields/terms/docs/positions on the
- * fly. This method will not return null.
+ * fly. This method will return null if the reader
+ * has no postings.
*
* <p><b>NOTE</b>: this is a slow way to access postings.
* It's better to get the sub-readers (using {@link
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Fri Sep 9 16:07:58 2011
@@ -164,6 +164,15 @@ public class _TestUtil {
}
}
+ // NOTE: only works for TMP and LMP!!
+ public static void setUseCompoundFile(MergePolicy mp, boolean v) {
+ if (mp instanceof TieredMergePolicy) {
+ ((TieredMergePolicy) mp).setUseCompoundFile(v);
+ } else if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(v);
+ }
+ }
+
/** start and end are BOTH inclusive */
public static int nextInt(Random r, int start, int end) {
return start + r.nextInt(end-start+1);
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Fri Sep 9 16:07:58 2011
@@ -1117,6 +1117,7 @@ public class TestIndexWriter extends Luc
IndexWriterConfig conf = newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2);
w = new IndexWriter(dir, conf);
+ w.setInfoStream(VERBOSE ? System.out : null);
Document doc = new Document();
doc.add(newField("field", "some text contents", storedTextType));
Modified: lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetVectorHighlightTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetVectorHighlightTask.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetVectorHighlightTask.java (original)
+++ lucene/dev/branches/flexscoring/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetVectorHighlightTask.java Fri Sep 9 16:07:58 2011
@@ -95,11 +95,12 @@ public class SearchTravRetVectorHighligh
@Override
protected BenchmarkHighlighter getBenchmarkHighlighter(Query q){
highlighter = new FastVectorHighlighter( false, false );
- final FieldQuery fq = highlighter.getFieldQuery( q );
+ final Query myq = q;
return new BenchmarkHighlighter(){
@Override
public int doHighlight(IndexReader reader, int doc, String field,
Document document, Analyzer analyzer, String text) throws Exception {
+ final FieldQuery fq = highlighter.getFieldQuery( myq, reader);
String[] fragments = highlighter.getBestFragments(fq, reader, doc, field, fragSize, maxFrags);
return fragments != null ? fragments.length : 0;
}
Modified: lucene/dev/branches/flexscoring/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/CHANGES.txt?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/CHANGES.txt (original)
+++ lucene/dev/branches/flexscoring/solr/CHANGES.txt Fri Sep 9 16:07:58 2011
@@ -321,6 +321,13 @@ Documentation
================== 3.5.0 ==================
+Bug Fixes
+----------------------
+* SOLR-2748: The CommitTracker used for commitWith or autoCommit by maxTime
+ could commit too frequently and could block adds until a new seaercher was
+ registered. (yonik)
+
+
Other Changes
----------------------
@@ -328,6 +335,8 @@ Documentation
ints masquerading as booleans. Preferred constructor now accepts a single int
bitfield (Chris Male)
+
+
================== 3.4.0 ==================
Upgrading from Solr 3.3
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java Fri Sep 9 16:07:58 2011
@@ -364,7 +364,7 @@ public class DefaultSolrHighlighter exte
// FVH cannot process hl.requireFieldMatch parameter per-field basis
params.getBool( HighlightParams.FIELD_MATCH, false ) );
fvh.setPhraseLimit(params.getInt(HighlightParams.PHRASE_LIMIT, Integer.MAX_VALUE));
- FieldQuery fieldQuery = fvh.getFieldQuery( query );
+ FieldQuery fieldQuery = fvh.getFieldQuery( query, searcher.getIndexReader() );
// Highlight each document
DocIterator iterator = docs.iterator();
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/update/CommitTracker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/update/CommitTracker.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/update/CommitTracker.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/update/CommitTracker.java Fri Sep 9 16:07:58 2011
@@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest;
@@ -43,7 +44,7 @@ final class CommitTracker implements Run
protected final static Logger log = LoggerFactory.getLogger(CommitTracker.class);
// scheduler delay for maxDoc-triggered autocommits
- public final int DOC_COMMIT_DELAY_MS = 250;
+ public final int DOC_COMMIT_DELAY_MS = 1;
// settings, not final so we can change them in testing
private int docsUpperBound;
@@ -56,8 +57,7 @@ final class CommitTracker implements Run
// state
private AtomicLong docsSinceCommit = new AtomicLong(0);
private AtomicInteger autoCommitCount = new AtomicInteger(0);
- private volatile long lastAddedTime = -1;
-
+
private final SolrCore core;
private final boolean softCommit;
@@ -88,20 +88,38 @@ final class CommitTracker implements Run
}
/** schedule individual commits */
- public synchronized void scheduleCommitWithin(long commitMaxTime) {
+ public void scheduleCommitWithin(long commitMaxTime) {
_scheduleCommitWithin(commitMaxTime);
}
-
- private synchronized void _scheduleCommitWithin(long commitMaxTime) {
- // Check if there is a commit already scheduled for longer then this time
- if (pending != null
- && pending.getDelay(TimeUnit.MILLISECONDS) >= commitMaxTime) {
- pending.cancel(false);
- pending = null;
- }
-
- // schedule a new commit
- if (pending == null) {
+
+ private void _scheduleCommitWithin(long commitMaxTime) {
+ if (commitMaxTime <= 0) return;
+ synchronized (this) {
+ if (pending != null && pending.getDelay(TimeUnit.MILLISECONDS) <= commitMaxTime) {
+ // There is already a pending commit that will happen first, so
+ // nothing else to do here.
+ // log.info("###returning since getDelay()==" + pending.getDelay(TimeUnit.MILLISECONDS) + " less than " + commitMaxTime);
+
+ return;
+ }
+
+ if (pending != null) {
+ // we need to schedule a commit to happen sooner than the existing one,
+ // so lets try to cancel the existing one first.
+ boolean canceled = pending.cancel(false);
+ if (!canceled) {
+ // It looks like we can't cancel... it must have just started running!
+ // this is possible due to thread scheduling delays and a low commitMaxTime.
+ // Nothing else to do since we obviously can't schedule our commit *before*
+ // the one that just started running (or has just completed).
+ // log.info("###returning since cancel failed");
+ return;
+ }
+ }
+
+ // log.info("###scheduling for " + commitMaxTime);
+
+ // schedule our new commit
pending = scheduler.schedule(this, commitMaxTime, TimeUnit.MILLISECONDS);
}
}
@@ -109,14 +127,15 @@ final class CommitTracker implements Run
/**
* Indicate that documents have been added
*/
- public boolean addedDocument(int commitWithin) {
- docsSinceCommit.incrementAndGet();
- lastAddedTime = System.currentTimeMillis();
- boolean triggered = false;
- // maxDocs-triggered autoCommit
- if (docsUpperBound > 0 && (docsSinceCommit.get() > docsUpperBound)) {
- _scheduleCommitWithin(DOC_COMMIT_DELAY_MS);
- triggered = true;
+ public void addedDocument(int commitWithin) {
+ // maxDocs-triggered autoCommit. Use == instead of > so we only trigger once on the way up
+ if (docsUpperBound > 0) {
+ long docs = docsSinceCommit.incrementAndGet();
+ if (docs == docsUpperBound + 1) {
+ // reset the count here instead of run() so we don't miss other documents being added
+ docsSinceCommit.set(0);
+ _scheduleCommitWithin(DOC_COMMIT_DELAY_MS);
+ }
}
// maxTime-triggered autoCommit
@@ -124,33 +143,31 @@ final class CommitTracker implements Run
if (ctime > 0) {
_scheduleCommitWithin(ctime);
- triggered = true;
}
-
- return triggered;
}
- /** Inform tracker that a commit has occurred, cancel any pending commits */
+ /** Inform tracker that a commit has occurred */
public void didCommit() {
- if (pending != null) {
- pending.cancel(false);
- pending = null; // let it start another one
- }
- docsSinceCommit.set(0);
}
/** Inform tracker that a rollback has occurred, cancel any pending commits */
public void didRollback() {
- if (pending != null) {
- pending.cancel(false);
- pending = null; // let it start another one
+ synchronized (this) {
+ if (pending != null) {
+ pending.cancel(false);
+ pending = null; // let it start another one
+ }
+ docsSinceCommit.set(0);
}
- docsSinceCommit.set(0);
}
/** This is the worker part for the ScheduledFuture **/
- public synchronized void run() {
- long started = System.currentTimeMillis();
+ public void run() {
+ synchronized (this) {
+ // log.info("###start commit. pending=null");
+ pending = null; // allow a new commit to be scheduled
+ }
+
SolrQueryRequest req = new LocalSolrQueryRequest(core,
new ModifiableSolrParams());
try {
@@ -158,25 +175,19 @@ final class CommitTracker implements Run
command.waitSearcher = waitSearcher;
command.softCommit = softCommit;
// no need for command.maxOptimizeSegments = 1; since it is not optimizing
- core.getUpdateHandler().commit(command);
+
+ // we increment this *before* calling commit because it was causing a race
+ // in the tests (the new searcher was registered and the test proceeded
+ // to check the commit count before we had incremented it.)
autoCommitCount.incrementAndGet();
+
+ core.getUpdateHandler().commit(command);
} catch (Exception e) {
- log.error("auto commit error...");
- e.printStackTrace();
+ SolrException.log(log, "auto commit error...", e);
} finally {
- pending = null;
+ // log.info("###done committing");
req.close();
}
-
- // check if docs have been submitted since the commit started
- if (lastAddedTime > started) {
- if (docsUpperBound > 0 && docsSinceCommit.get() > docsUpperBound) {
- pending = scheduler.schedule(this, 100, TimeUnit.MILLISECONDS);
- } else if (timeUpperBound > 0) {
- pending = scheduler.schedule(this, timeUpperBound,
- TimeUnit.MILLISECONDS);
- }
- }
}
// to facilitate testing: blocks if called during commit
Modified: lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Fri Sep 9 16:07:58 2011
@@ -491,7 +491,7 @@ public class DirectUpdateHandler2 extend
NamedList lst = new SimpleOrderedMap();
lst.add("commits", commitCommands.get());
if (commitTracker.getTimeUpperBound() > 0) {
- lst.add("autocommit maxDocs", commitTracker.getTimeUpperBound());
+ lst.add("autocommit maxDocs", commitTracker.getDocsUpperBound());
}
if (commitTracker.getTimeUpperBound() > 0) {
lst.add("autocommit maxTime", "" + commitTracker.getTimeUpperBound() + "ms");
Modified: lucene/dev/branches/flexscoring/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java?rev=1167270&r1=1167269&r2=1167270&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java (original)
+++ lucene/dev/branches/flexscoring/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java Fri Sep 9 16:07:58 2011
@@ -33,26 +33,28 @@ import org.apache.solr.response.SolrQuer
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.NewSearcherListener.TriggerOn;
import org.apache.solr.util.AbstractSolrTestCase;
+import org.apache.solr.util.RefCounted;
class NewSearcherListener implements SolrEventListener {
enum TriggerOn {Both, Soft, Hard}
-
+
private volatile boolean triggered = false;
private volatile TriggerOn lastType;
private volatile TriggerOn triggerOnType;
-
+ private volatile SolrIndexSearcher newSearcher;
+
public NewSearcherListener() {
this(TriggerOn.Both);
}
-
+
public NewSearcherListener(TriggerOn type) {
this.triggerOnType = type;
}
-
+
@Override
public void init(NamedList args) {}
-
+
@Override
public void newSearcher(SolrIndexSearcher newSearcher,
SolrIndexSearcher currentSearcher) {
@@ -63,33 +65,41 @@ class NewSearcherListener implements Sol
} else if (triggerOnType == TriggerOn.Both) {
triggered = true;
}
+ this.newSearcher = newSearcher;
+ // log.info("TEST: newSearcher event: triggered="+triggered+" newSearcher="+newSearcher);
}
-
+
@Override
public void postCommit() {
lastType = TriggerOn.Hard;
}
-
+
@Override
public void postSoftCommit() {
lastType = TriggerOn.Soft;
}
-
+
public void reset() {
triggered = false;
+ // log.info("TEST: trigger reset");
}
-
+
boolean waitForNewSearcher(int timeout) {
long timeoutTime = System.currentTimeMillis() + timeout;
while (System.currentTimeMillis() < timeoutTime) {
if (triggered) {
- return true;
+ // check if the new searcher has been registered yet
+ RefCounted<SolrIndexSearcher> registeredSearcherH = newSearcher.getCore().getSearcher();
+ SolrIndexSearcher registeredSearcher = registeredSearcherH.get();
+ registeredSearcherH.decref();
+ if (registeredSearcher == newSearcher) return true;
+ // log.info("TEST: waiting for searcher " + newSearcher + " to be registered. current=" + registeredSearcher);
}
-
+
try {
Thread.sleep(250);
} catch (InterruptedException e) {}
-
+
}
return false;
}
@@ -102,6 +112,19 @@ public class AutoCommitTest extends Abst
@Override
public String getSolrConfigFile() { return "solrconfig.xml"; }
+ public static void verbose(Object... args) {
+ if (!VERBOSE) return;
+ StringBuilder sb = new StringBuilder("###TEST:");
+ sb.append(Thread.currentThread().getName());
+ sb.append(':');
+ for (Object o : args) {
+ sb.append(' ');
+ sb.append(o.toString());
+ }
+ log.info(sb.toString());
+ // System.out.println(sb.toString());
+ }
+
/**
* Take a string and make it an iterable ContentStream
*
@@ -345,6 +368,7 @@ public class AutoCommitTest extends Abst
// Check it it is in the index
assertQ("shouldn't find any", req("id:529") ,"//result[@numFound=0]" );
+ assertEquals(0, softTracker.getCommitCount());
// Wait longer than the autocommit time
assertTrue(trigger.waitForNewSearcher(30000));
@@ -357,16 +381,22 @@ public class AutoCommitTest extends Abst
assertQ("should find one", req("id:529") ,"//result[@numFound=1]" );
// But not this one
assertQ("should find none", req("id:530") ,"//result[@numFound=0]" );
-
+ verbose("###about to delete 529");
// Delete the document
- assertU( delI("529") );
+ assertU(delI("529"));
assertQ("deleted, but should still be there", req("id:529") ,"//result[@numFound=1]" );
// Wait longer than the autocommit time
+ verbose("###starting to wait for new searcher. softTracker.getCommitCount()==",softTracker.getCommitCount());
assertTrue(trigger.waitForNewSearcher(15000));
trigger.reset();
+ verbose("###done waiting for new searcher. softTracker.getCommitCount()==",softTracker.getCommitCount());
+
+ // what's the point of this update?
req.setContentStreams( toContentStreams(
adoc("id", "550", "field_t", "what's inside?", "subject", "info"), null ) );
handler.handleRequest( req, rsp );
+
+
assertEquals( 2, softTracker.getCommitCount() );
assertQ("deleted and time has passed", req("id:529") ,"//result[@numFound=0]" );