You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/10/20 14:44:39 UTC
svn commit: r1025539 [2/10] - in /lucene/dev/branches/docvalues: ./ lucene/
lucene/contrib/
lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/
lucene/contrib/highlighter/src/test/
lucene/contrib/instantiated/src/test/org/apache...
Modified: lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FieldBoostMapAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FieldBoostMapAttributeImpl.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FieldBoostMapAttributeImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FieldBoostMapAttributeImpl.java Wed Oct 20 12:44:28 2010
@@ -38,18 +38,18 @@ public class FieldBoostMapAttributeImpl
private static final long serialVersionUID = -2104763012523049527L;
- private Map<CharSequence, Float> boosts = new LinkedHashMap<CharSequence, Float>();
+ private Map<String, Float> boosts = new LinkedHashMap<String, Float>();
public FieldBoostMapAttributeImpl() {
// empty constructor
}
- public void setFieldBoostMap(Map<CharSequence, Float> boosts) {
+ public void setFieldBoostMap(Map<String, Float> boosts) {
this.boosts = boosts;
}
- public Map<CharSequence, Float> getFieldBoostMap() {
+ public Map<String, Float> getFieldBoostMap() {
return this.boosts;
}
Modified: lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/StandardSyntaxParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/StandardSyntaxParser.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/StandardSyntaxParser.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/StandardSyntaxParser.java Wed Oct 20 12:44:28 2010
@@ -378,7 +378,7 @@ public class StandardSyntaxParser implem
boolean regexp = false;
QueryNode q =null;
ParametricQueryNode qLower, qUpper;
- float defaultMinSimilarity = 0.5f;
+ float defaultMinSimilarity = org.apache.lucene.search.FuzzyQuery.defaultMinSimilarity;
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case TERM:
case REGEXPTERM:
Modified: lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/StandardSyntaxParser.jj
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/StandardSyntaxParser.jj?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/StandardSyntaxParser.jj (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/StandardSyntaxParser.jj Wed Oct 20 12:44:28 2010
@@ -378,7 +378,7 @@ QueryNode Term(CharSequence field) : {
boolean regexp = false;
QueryNode q =null;
ParametricQueryNode qLower, qUpper;
- float defaultMinSimilarity = 0.5f;
+ float defaultMinSimilarity = org.apache.lucene.search.FuzzyQuery.defaultMinSimilarity;
}
{
(
Modified: lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/StandardQueryNodeProcessorPipeline.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/StandardQueryNodeProcessorPipeline.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/StandardQueryNodeProcessorPipeline.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/StandardQueryNodeProcessorPipeline.java Wed Oct 20 12:44:28 2010
@@ -48,23 +48,23 @@ public class StandardQueryNodeProcessorP
public StandardQueryNodeProcessorPipeline(QueryConfigHandler queryConfig) {
super(queryConfig);
- addProcessor(new WildcardQueryNodeProcessor());
- addProcessor(new MultiFieldQueryNodeProcessor());
- addProcessor(new FuzzyQueryNodeProcessor());
- addProcessor(new MatchAllDocsQueryNodeProcessor());
- addProcessor(new LowercaseExpandedTermsQueryNodeProcessor());
- addProcessor(new ParametricRangeQueryNodeProcessor());
- addProcessor(new AllowLeadingWildcardProcessor());
- addProcessor(new AnalyzerQueryNodeProcessor());
- addProcessor(new PhraseSlopQueryNodeProcessor());
- addProcessor(new GroupQueryNodeProcessor());
- addProcessor(new NoChildOptimizationQueryNodeProcessor());
- addProcessor(new RemoveDeletedQueryNodesProcessor());
- addProcessor(new RemoveEmptyNonLeafQueryNodeProcessor());
- addProcessor(new BooleanSingleChildOptimizationQueryNodeProcessor());
- addProcessor(new DefaultPhraseSlopQueryNodeProcessor());
- addProcessor(new BoostQueryNodeProcessor());
- addProcessor(new MultiTermRewriteMethodProcessor());
+ add(new WildcardQueryNodeProcessor());
+ add(new MultiFieldQueryNodeProcessor());
+ add(new FuzzyQueryNodeProcessor());
+ add(new MatchAllDocsQueryNodeProcessor());
+ add(new LowercaseExpandedTermsQueryNodeProcessor());
+ add(new ParametricRangeQueryNodeProcessor());
+ add(new AllowLeadingWildcardProcessor());
+ add(new AnalyzerQueryNodeProcessor());
+ add(new PhraseSlopQueryNodeProcessor());
+ add(new GroupQueryNodeProcessor());
+ add(new NoChildOptimizationQueryNodeProcessor());
+ add(new RemoveDeletedQueryNodesProcessor());
+ add(new RemoveEmptyNonLeafQueryNodeProcessor());
+ add(new BooleanSingleChildOptimizationQueryNodeProcessor());
+ add(new DefaultPhraseSlopQueryNodeProcessor());
+ add(new BoostQueryNodeProcessor());
+ add(new MultiTermRewriteMethodProcessor());
}
}
Modified: lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java Wed Oct 20 12:44:28 2010
@@ -43,7 +43,6 @@ import org.apache.lucene.search.BooleanC
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.automaton.BasicAutomata;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.junit.runner.RunWith;
import java.io.IOException;
import java.io.Reader;
@@ -51,7 +50,6 @@ import java.text.DateFormat;
import java.util.Calendar;
import java.util.GregorianCalendar;
-@RunWith(LuceneTestCase.LocalizedTestCaseRunner.class)
public class TestPrecedenceQueryParser extends LuceneTestCase {
public static Analyzer qpAnalyzer = new QPTestAnalyzer();
Modified: lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/spans/TestSpanQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/spans/TestSpanQueryParser.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/spans/TestSpanQueryParser.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/spans/TestSpanQueryParser.java Wed Oct 20 12:44:28 2010
@@ -117,11 +117,9 @@ public class TestSpanQueryParser extends
this.spanProcessorPipeline
.setQueryConfigHandler(this.spanQueryConfigHandler);
- this.spanProcessorPipeline.addProcessor(new WildcardQueryNodeProcessor());
- this.spanProcessorPipeline
- .addProcessor(new SpansValidatorQueryNodeProcessor());
- this.spanProcessorPipeline
- .addProcessor(new UniqueFieldQueryNodeProcessor());
+ this.spanProcessorPipeline.add(new WildcardQueryNodeProcessor());
+ this.spanProcessorPipeline.add(new SpansValidatorQueryNodeProcessor());
+ this.spanProcessorPipeline.add(new UniqueFieldQueryNodeProcessor());
}
Modified: lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/spans/TestSpanQueryParserSimpleSample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/spans/TestSpanQueryParserSimpleSample.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/spans/TestSpanQueryParserSimpleSample.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/spans/TestSpanQueryParserSimpleSample.java Wed Oct 20 12:44:28 2010
@@ -113,9 +113,9 @@ public class TestSpanQueryParserSimpleSa
QueryNodeProcessorPipeline spanProcessorPipeline = new QueryNodeProcessorPipeline(
spanQueryConfigHandler);
// @see SpansValidatorQueryNodeProcessor
- spanProcessorPipeline.addProcessor(new SpansValidatorQueryNodeProcessor());
+ spanProcessorPipeline.add(new SpansValidatorQueryNodeProcessor());
// @see UniqueFieldQueryNodeProcessor
- spanProcessorPipeline.addProcessor(new UniqueFieldQueryNodeProcessor());
+ spanProcessorPipeline.add(new UniqueFieldQueryNodeProcessor());
// print to show out the QueryNode tree before being processed
if (VERBOSE) System.out.println(queryTree);
Modified: lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQPHelper.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQPHelper.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQPHelper.java Wed Oct 20 12:44:28 2010
@@ -99,7 +99,7 @@ public class TestMultiFieldQPHelper exte
assertEquals("((b:one t:one)^2.0) (b:two t:two)", q.toString());
q = mfqp.parse("one~ two", null);
- assertEquals("(b:one~0.5 t:one~0.5) (b:two t:two)", q.toString());
+ assertEquals("(b:one~2.0 t:one~2.0) (b:two t:two)", q.toString());
q = mfqp.parse("one~0.8 two^2", null);
assertEquals("(b:one~0.8 t:one~0.8) ((b:two t:two)^2.0)", q.toString());
@@ -143,7 +143,7 @@ public class TestMultiFieldQPHelper exte
}
public void testBoostsSimple() throws Exception {
- Map<CharSequence,Float> boosts = new HashMap<CharSequence,Float>();
+ Map<String,Float> boosts = new HashMap<String,Float>();
boosts.put("b", Float.valueOf(5));
boosts.put("t", Float.valueOf(10));
String[] fields = { "b", "t" };
@@ -310,7 +310,7 @@ public class TestMultiFieldQPHelper exte
q = parser.parse("bla*", null);
assertEquals("f1:bla* f2:bla* f3:bla*", q.toString());
q = parser.parse("bla~", null);
- assertEquals("f1:bla~0.5 f2:bla~0.5 f3:bla~0.5", q.toString());
+ assertEquals("f1:bla~2.0 f2:bla~2.0 f3:bla~2.0", q.toString());
q = parser.parse("[a TO c]", null);
assertEquals("f1:[a TO c] f2:[a TO c] f3:[a TO c]", q.toString());
}
Modified: lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQueryParserWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQueryParserWrapper.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQueryParserWrapper.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQueryParserWrapper.java Wed Oct 20 12:44:28 2010
@@ -100,7 +100,7 @@ public class TestMultiFieldQueryParserWr
assertEquals("((b:one t:one)^2.0) (b:two t:two)", q.toString());
q = mfqp.parse("one~ two");
- assertEquals("(b:one~0.5 t:one~0.5) (b:two t:two)", q.toString());
+ assertEquals("(b:one~2.0 t:one~2.0) (b:two t:two)", q.toString());
q = mfqp.parse("one~0.8 two^2");
assertEquals("(b:one~0.8 t:one~0.8) ((b:two t:two)^2.0)", q.toString());
@@ -144,7 +144,7 @@ public class TestMultiFieldQueryParserWr
}
public void testBoostsSimple() throws Exception {
- Map<CharSequence,Float> boosts = new HashMap<CharSequence,Float>();
+ Map<String,Float> boosts = new HashMap<String,Float>();
boosts.put("b", Float.valueOf(5));
boosts.put("t", Float.valueOf(10));
String[] fields = { "b", "t" };
@@ -314,7 +314,7 @@ public class TestMultiFieldQueryParserWr
q = parser.parse("bla*");
assertEquals("f1:bla* f2:bla* f3:bla*", q.toString());
q = parser.parse("bla~");
- assertEquals("f1:bla~0.5 f2:bla~0.5 f3:bla~0.5", q.toString());
+ assertEquals("f1:bla~2.0 f2:bla~2.0 f3:bla~2.0", q.toString());
q = parser.parse("[a TO c]");
assertEquals("f1:[a TO c] f2:[a TO c] f3:[a TO c]", q.toString());
}
Modified: lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java Wed Oct 20 12:44:28 2010
@@ -76,7 +76,6 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.automaton.BasicAutomata;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.automaton.RegExp;
-import org.junit.runner.RunWith;
/**
* This test case is a copy of the core Lucene query parser test, it was adapted
@@ -84,7 +83,6 @@ import org.junit.runner.RunWith;
*
* Tests QueryParser.
*/
-@RunWith(LuceneTestCase.LocalizedTestCaseRunner.class)
public class TestQPHelper extends LuceneTestCase {
public static Analyzer qpAnalyzer = new QPTestAnalyzer();
@@ -140,7 +138,7 @@ public class TestQPHelper extends Lucene
public static class QPTestParser extends StandardQueryParser {
public QPTestParser(Analyzer a) {
((QueryNodeProcessorPipeline)getQueryNodeProcessor())
- .addProcessor(new QPTestParserQueryNodeProcessor());
+ .add(new QPTestParserQueryNodeProcessor());
this.setAnalyzer(a);
}
@@ -502,12 +500,12 @@ public class TestQPHelper extends Lucene
public void testWildcard() throws Exception {
assertQueryEquals("term*", null, "term*");
assertQueryEquals("term*^2", null, "term*^2.0");
- assertQueryEquals("term~", null, "term~0.5");
+ assertQueryEquals("term~", null, "term~2.0");
assertQueryEquals("term~0.7", null, "term~0.7");
- assertQueryEquals("term~^2", null, "term~0.5^2.0");
+ assertQueryEquals("term~^3", null, "term~2.0^3.0");
- assertQueryEquals("term^2~", null, "term~0.5^2.0");
+ assertQueryEquals("term^3~", null, "term~2.0^3.0");
assertQueryEquals("term*germ", null, "term*germ");
assertQueryEquals("term*germ^3", null, "term*germ^3.0");
@@ -519,7 +517,7 @@ public class TestQPHelper extends Lucene
assertEquals(0.7f, fq.getMinSimilarity(), 0.1f);
assertEquals(FuzzyQuery.defaultPrefixLength, fq.getPrefixLength());
fq = (FuzzyQuery) getQuery("term~", null);
- assertEquals(0.5f, fq.getMinSimilarity(), 0.1f);
+ assertEquals(2.0f, fq.getMinSimilarity(), 0.1f);
assertEquals(FuzzyQuery.defaultPrefixLength, fq.getPrefixLength());
assertQueryNodeException("term~1.1"); // value > 1, throws exception
@@ -555,9 +553,9 @@ public class TestQPHelper extends Lucene
assertWildcardQueryEquals("TE?M", false, "TE?M");
assertWildcardQueryEquals("Te?m*gerM", false, "Te?m*gerM");
// Fuzzy queries:
- assertWildcardQueryEquals("Term~", "term~0.5");
- assertWildcardQueryEquals("Term~", true, "term~0.5");
- assertWildcardQueryEquals("Term~", false, "Term~0.5");
+ assertWildcardQueryEquals("Term~", "term~2.0");
+ assertWildcardQueryEquals("Term~", true, "term~2.0");
+ assertWildcardQueryEquals("Term~", false, "Term~2.0");
// Range queries:
// TODO: implement this on QueryParser
@@ -859,10 +857,10 @@ public class TestQPHelper extends Lucene
assertQueryEquals("a:b\\\\?c", a, "a:b\\?c");
- assertQueryEquals("a:b\\-c~", a, "a:b-c~0.5");
- assertQueryEquals("a:b\\+c~", a, "a:b+c~0.5");
- assertQueryEquals("a:b\\:c~", a, "a:b:c~0.5");
- assertQueryEquals("a:b\\\\c~", a, "a:b\\c~0.5");
+ assertQueryEquals("a:b\\-c~", a, "a:b-c~2.0");
+ assertQueryEquals("a:b\\+c~", a, "a:b+c~2.0");
+ assertQueryEquals("a:b\\:c~", a, "a:b:c~2.0");
+ assertQueryEquals("a:b\\\\c~", a, "a:b\\c~2.0");
// TODO: implement Range queries on QueryParser
assertQueryEquals("[ a\\- TO a\\+ ]", null, "[a- TO a+]");
Modified: lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java Wed Oct 20 12:44:28 2010
@@ -21,11 +21,9 @@ import java.io.IOException;
import java.io.Reader;
import java.text.Collator;
import java.text.DateFormat;
-import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
-import java.util.HashSet;
import java.util.List;
import java.util.Locale;
@@ -72,7 +70,6 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.automaton.BasicAutomata;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.automaton.RegExp;
-import org.junit.runner.RunWith;
/**
* This test case is a copy of the core Lucene query parser test, it was adapted
@@ -83,7 +80,6 @@ import org.junit.runner.RunWith;
* @deprecated this entire test case tests QueryParserWrapper which is
* deprecated. When QPW is gone, so will the test.
*/
-@RunWith(LuceneTestCase.LocalizedTestCaseRunner.class)
@Deprecated
public class TestQueryParserWrapper extends LuceneTestCase {
@@ -143,9 +139,9 @@ public class TestQueryParserWrapper exte
QueryNodeProcessorPipeline newProcessorPipeline = new QueryNodeProcessorPipeline(
getQueryProcessor().getQueryConfigHandler());
- newProcessorPipeline.addProcessor(new WildcardQueryNodeProcessor());
- newProcessorPipeline.addProcessor(new QPTestParserQueryNodeProcessor());
- newProcessorPipeline.addProcessor(getQueryProcessor());
+ newProcessorPipeline.add(new WildcardQueryNodeProcessor());
+ newProcessorPipeline.add(new QPTestParserQueryNodeProcessor());
+ newProcessorPipeline.add(getQueryProcessor());
setQueryProcessor(newProcessorPipeline);
@@ -505,12 +501,12 @@ public class TestQueryParserWrapper exte
public void testWildcard() throws Exception {
assertQueryEquals("term*", null, "term*");
assertQueryEquals("term*^2", null, "term*^2.0");
- assertQueryEquals("term~", null, "term~0.5");
+ assertQueryEquals("term~", null, "term~2.0");
assertQueryEquals("term~0.7", null, "term~0.7");
- assertQueryEquals("term~^2", null, "term~0.5^2.0");
+ assertQueryEquals("term~^3", null, "term~2.0^3.0");
- assertQueryEquals("term^2~", null, "term~0.5^2.0");
+ assertQueryEquals("term^3~", null, "term~2.0^3.0");
assertQueryEquals("term*germ", null, "term*germ");
assertQueryEquals("term*germ^3", null, "term*germ^3.0");
@@ -522,7 +518,7 @@ public class TestQueryParserWrapper exte
assertEquals(0.7f, fq.getMinSimilarity(), 0.1f);
assertEquals(FuzzyQuery.defaultPrefixLength, fq.getPrefixLength());
fq = (FuzzyQuery) getQuery("term~", null);
- assertEquals(0.5f, fq.getMinSimilarity(), 0.1f);
+ assertEquals(2.0f, fq.getMinSimilarity(), 0.1f);
assertEquals(FuzzyQuery.defaultPrefixLength, fq.getPrefixLength());
assertParseException("term~1.1"); // value > 1, throws exception
@@ -558,9 +554,9 @@ public class TestQueryParserWrapper exte
assertWildcardQueryEquals("TE?M", false, "TE?M");
assertWildcardQueryEquals("Te?m*gerM", false, "Te?m*gerM");
// Fuzzy queries:
- assertWildcardQueryEquals("Term~", "term~0.5");
- assertWildcardQueryEquals("Term~", true, "term~0.5");
- assertWildcardQueryEquals("Term~", false, "Term~0.5");
+ assertWildcardQueryEquals("Term~", "term~2.0");
+ assertWildcardQueryEquals("Term~", true, "term~2.0");
+ assertWildcardQueryEquals("Term~", false, "Term~2.0");
// Range queries:
// TODO: implement this on QueryParser
@@ -854,10 +850,10 @@ public class TestQueryParserWrapper exte
assertQueryEquals("a:b\\\\?c", a, "a:b\\?c");
- assertQueryEquals("a:b\\-c~", a, "a:b-c~0.5");
- assertQueryEquals("a:b\\+c~", a, "a:b+c~0.5");
- assertQueryEquals("a:b\\:c~", a, "a:b:c~0.5");
- assertQueryEquals("a:b\\\\c~", a, "a:b\\c~0.5");
+ assertQueryEquals("a:b\\-c~", a, "a:b-c~2.0");
+ assertQueryEquals("a:b\\+c~", a, "a:b+c~2.0");
+ assertQueryEquals("a:b\\:c~", a, "a:b:c~2.0");
+ assertQueryEquals("a:b\\\\c~", a, "a:b\\c~2.0");
// TODO: implement Range queries on QueryParser
assertQueryEquals("[ a\\- TO a\\+ ]", null, "[a- TO a+]");
Modified: lucene/dev/branches/docvalues/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java Wed Oct 20 12:44:28 2010
@@ -30,6 +30,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.Term;
import org.apache.lucene.search.FuzzyTermsEnum;
import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.automaton.LevenshteinAutomata;
@@ -387,7 +388,10 @@ public class DirectSpellChecker {
private Collection<ScoreTerm> suggestSimilar(Term term, int numSug,
IndexReader ir, int docfreq, int editDistance, float accuracy) throws IOException {
- FuzzyTermsEnum e = new FuzzyTermsEnum(ir, term, editDistance, Math.max(minPrefix, editDistance-1));
+ AttributeSource atts = new AttributeSource();
+ MultiTermQuery.MaxNonCompetitiveBoostAttribute maxBoostAtt =
+ atts.addAttribute(MultiTermQuery.MaxNonCompetitiveBoostAttribute.class);
+ FuzzyTermsEnum e = new FuzzyTermsEnum(ir, atts, term, editDistance, Math.max(minPrefix, editDistance-1));
final PriorityQueue<ScoreTerm> stQueue = new PriorityQueue<ScoreTerm>();
BytesRef queryTerm = new BytesRef(term.text());
@@ -435,7 +439,7 @@ public class DirectSpellChecker {
stQueue.offer(st);
// possibly drop entries from queue
st = (stQueue.size() > numSug) ? stQueue.poll() : new ScoreTerm();
- boostAtt.setMaxNonCompetitiveBoost((stQueue.size() >= numSug) ? stQueue.peek().boost : Float.NEGATIVE_INFINITY);
+ maxBoostAtt.setMaxNonCompetitiveBoost((stQueue.size() >= numSug) ? stQueue.peek().boost : Float.NEGATIVE_INFINITY);
}
return stQueue;
Modified: lucene/dev/branches/docvalues/lucene/contrib/xml-query-parser/LuceneContribQuery.dtd
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/xml-query-parser/LuceneContribQuery.dtd?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/xml-query-parser/LuceneContribQuery.dtd (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/xml-query-parser/LuceneContribQuery.dtd Wed Oct 20 12:44:28 2010
@@ -64,11 +64,8 @@ Improves on FuzzyQuery by rewarding all
<!-- Controls the level of similarity required for fuzzy variants where 1 is identical and 0.5 is that the variant contains
half of the original's characters in the same order. Lower values produce more results but may take longer to execute due to
additional IO required to read matching document ids-->
-<!ATTLIST Field minSimilarity CDATA "0.5">
+<!ATTLIST Field minSimilarity CDATA "2.0">
<!-- Controls the minimum number of characters at the start of fuzzy variant words that must exactly match the original.
- A value of zero will require no minimum and the search software will effectively scan ALL terms from a to z looking for variations.
- This can incur high CPU overhead and a prefix length of just "1" will reduce this overhead to 1/26th of the original cost (assuming
- an even distribution of letters used from the alphabet).
-->
<!ATTLIST Field prefixLength CDATA "1">
<!-- fieldName must be defined here or is taken from the most immediate parent XML element that defines a "fieldName" attribute -->
Modified: lucene/dev/branches/docvalues/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/FuzzyLikeThisQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/FuzzyLikeThisQueryBuilder.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/FuzzyLikeThisQueryBuilder.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/FuzzyLikeThisQueryBuilder.java Wed Oct 20 12:44:28 2010
@@ -2,6 +2,7 @@ package org.apache.lucene.xmlparser.buil
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.search.FuzzyLikeThisQuery;
+import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.xmlparser.DOMUtils;
import org.apache.lucene.xmlparser.ParserException;
@@ -32,7 +33,7 @@ import org.w3c.dom.NodeList;
public class FuzzyLikeThisQueryBuilder implements QueryBuilder
{
int defaultMaxNumTerms=50;
- float defaultMinSimilarity=0.5f;
+ float defaultMinSimilarity=FuzzyQuery.defaultMinSimilarity;
int defaultPrefixLength=1;
boolean defaultIgnoreTF=false;
private Analyzer analyzer;
Modified: lucene/dev/branches/docvalues/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestQueryTemplateManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestQueryTemplateManager.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestQueryTemplateManager.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestQueryTemplateManager.java Wed Oct 20 12:44:28 2010
@@ -17,7 +17,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase;
-import org.junit.Assume;
+
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
@@ -75,7 +75,7 @@ public class TestQueryTemplateManager ex
// Sun 1.5 suffers from http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6240963
if (Constants.JAVA_VENDOR.startsWith("Sun") && Constants.JAVA_VERSION.startsWith("1.5")) {
String defLang = Locale.getDefault().getLanguage();
- Assume.assumeTrue(!defLang.equals("tr") && !defLang.equals("az"));
+ assumeFalse("Sun JRE 1.5 suffers from http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6240963 under Turkish locale", defLang.equals("tr") || defLang.equals("az"));
}
//Cache all the query templates we will be referring to.
QueryTemplateManager qtm=new QueryTemplateManager();
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct 20 12:44:28 2010
@@ -1,5 +1,6 @@
/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512
/lucene/dev/branches/preflexfixes/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:967125-979432
+/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:1021635-1024556,1025532-1025536
/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java:824912-931101
/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:909334,948516
/lucene/java/trunk/src/java/org/apache/lucene/analysis/Tokenizer.java:924483-924731,924781,925176-925462
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java Wed Oct 20 12:44:28 2010
@@ -203,7 +203,8 @@ public class CharTermAttributeImpl exten
public Object clone() {
CharTermAttributeImpl t = (CharTermAttributeImpl)super.clone();
// Do a deep clone
- t.termBuffer = termBuffer.clone();
+ t.termBuffer = new char[this.termLength];
+ System.arraycopy(this.termBuffer, 0, t.termBuffer, 0, this.termLength);
return t;
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java Wed Oct 20 12:44:28 2010
@@ -733,8 +733,6 @@ public class CheckIndex {
throw new RuntimeException("termCount mismatch " + termCount + " vs " + (status.termCount - termCountStart));
}
- termCount = status.termCount;
-
int seekCount = (int) Math.min(10000L, termCount);
if (seekCount > 0) {
BytesRef[] seekTerms = new BytesRef[seekCount];
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java Wed Oct 20 12:44:28 2010
@@ -106,7 +106,7 @@ class FixedSortedBytesImpl {
// first dump bytes data, recording address as we go
for(int i=0;i<count;i++) {
final int e = sortedEntries[i];
- final BytesRef bytes = hash.get(e);
+ final BytesRef bytes = hash.get(e, new BytesRef());
assert bytes.length == size;
datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length);
address[e] = 1+i;
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java Wed Oct 20 12:44:28 2010
@@ -109,7 +109,7 @@ class VarSortedBytesImpl {
offsets[i] = offset;
index[e] = 1 + i;
- final BytesRef bytes = hash.get(e);
+ final BytesRef bytes = hash.get(e, new BytesRef());
// TODO: we could prefix code...
datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length);
lastOffset = offset;
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/messages/NLS.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/messages/NLS.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/messages/NLS.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/messages/NLS.java Wed Oct 20 12:44:28 2010
@@ -45,8 +45,8 @@ import java.util.ResourceBundle;
*/
public class NLS {
- private static Map<String, Class<Object>> bundles =
- new HashMap<String, Class<Object>>(0);
+ private static Map<String, Class<? extends NLS>> bundles =
+ new HashMap<String, Class<? extends NLS>>(0);
protected NLS() {
// Do not instantiate
@@ -89,8 +89,7 @@ public class NLS {
* @param clazz
* where constants will reside
*/
- @SuppressWarnings("unchecked")
- protected static void initializeMessages(String bundleName, Class clazz) {
+ protected static void initializeMessages(String bundleName, Class<? extends NLS> clazz) {
try {
load(clazz);
if (!bundles.containsKey(bundleName))
@@ -106,7 +105,7 @@ public class NLS {
// slow resource checking
// need to loop thru all registered resource bundles
for (Iterator<String> it = bundles.keySet().iterator(); it.hasNext();) {
- Class<Object> clazz = bundles.get(it.next());
+ Class<? extends NLS> clazz = bundles.get(it.next());
ResourceBundle resourceBundle = ResourceBundle.getBundle(clazz.getName(),
locale);
if (resourceBundle != null) {
@@ -126,7 +125,7 @@ public class NLS {
/**
* @param clazz
*/
- private static void load(Class<Object> clazz) {
+ private static void load(Class<? extends NLS> clazz) {
final Field[] fieldArray = clazz.getDeclaredFields();
boolean isFieldAccessible = (clazz.getModifiers() & Modifier.PUBLIC) != 0;
@@ -145,7 +144,7 @@ public class NLS {
* @param isFieldAccessible
*/
private static void loadfieldValue(Field field, boolean isFieldAccessible,
- Class<Object> clazz) {
+ Class<? extends NLS> clazz) {
int MOD_EXPECTED = Modifier.PUBLIC | Modifier.STATIC;
int MOD_MASK = MOD_EXPECTED | Modifier.FINAL;
if ((field.getModifiers() & MOD_MASK) != MOD_EXPECTED)
@@ -168,7 +167,7 @@ public class NLS {
* @param key
* - Message Key
*/
- private static void validateMessage(String key, Class<Object> clazz) {
+ private static void validateMessage(String key, Class<? extends NLS> clazz) {
// Test if the message is present in the resource bundle
try {
ResourceBundle resourceBundle = ResourceBundle.getBundle(clazz.getName(),
@@ -192,13 +191,12 @@ public class NLS {
/*
* Make a class field accessible
*/
- @SuppressWarnings("unchecked")
private static void makeAccessible(final Field field) {
if (System.getSecurityManager() == null) {
field.setAccessible(true);
} else {
- AccessController.doPrivileged(new PrivilegedAction() {
- public Object run() {
+ AccessController.doPrivileged(new PrivilegedAction<Void>() {
+ public Void run() {
field.setAccessible(true);
return null;
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/AutomatonQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/AutomatonQuery.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/AutomatonQuery.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/AutomatonQuery.java Wed Oct 20 12:44:28 2010
@@ -24,6 +24,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.ByteRunAutomaton;
@@ -85,7 +86,7 @@ public class AutomatonQuery extends Mult
}
@Override
- protected TermsEnum getTermsEnum(IndexReader reader) throws IOException {
+ protected TermsEnum getTermsEnum(IndexReader reader, AttributeSource atts) throws IOException {
// matches nothing
if (BasicOperations.isEmpty(automaton)) {
return TermsEnum.EMPTY;
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/BooleanScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/BooleanScorer.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/BooleanScorer.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/BooleanScorer.java Wed Oct 20 12:44:28 2010
@@ -69,7 +69,7 @@ final class BooleanScorer extends Scorer
}
@Override
- public final void collect(final int doc) throws IOException {
+ public void collect(final int doc) throws IOException {
final BucketTable table = bucketTable;
final int i = doc & BucketTable.MASK;
Bucket bucket = table.buckets[i];
@@ -159,7 +159,7 @@ final class BooleanScorer extends Scorer
return new BooleanScorerCollector(mask, this);
}
- public final int size() { return SIZE; }
+ public int size() { return SIZE; }
}
static final class SubScorer {
@@ -320,14 +320,10 @@ final class BooleanScorer extends Scorer
more = false;
end += BucketTable.SIZE;
for (SubScorer sub = scorers; sub != null; sub = sub.next) {
- Scorer scorer = sub.scorer;
- sub.collector.setScorer(scorer);
- int doc = scorer.docID();
- while (doc < end) {
- sub.collector.collect(doc);
- doc = scorer.nextDoc();
+ int subScorerDocID = sub.scorer.docID();
+ if (subScorerDocID != NO_MORE_DOCS) {
+ more |= sub.scorer.score(sub.collector, end, subScorerDocID);
}
- more |= (doc != NO_MORE_DOCS);
}
} while (bucketTable.first != null || more);
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java Wed Oct 20 12:44:28 2010
@@ -163,6 +163,7 @@ public class FieldCacheImpl implements F
}
}
+ @SuppressWarnings("unchecked")
public Object get(IndexReader reader, Entry<T> key) throws IOException {
Map<Entry<T>,Object> innerCache;
Object value;
@@ -273,6 +274,7 @@ public class FieldCacheImpl implements F
return getBytes(reader, field, new ByteValuesCreator(field, parser)).values;
}
+ @SuppressWarnings("unchecked")
public ByteValues getBytes(IndexReader reader, String field, EntryCreator<ByteValues> creator ) throws IOException
{
return (ByteValues)caches.get(Byte.TYPE).get(reader, new Entry(field, creator));
@@ -288,6 +290,7 @@ public class FieldCacheImpl implements F
return getShorts(reader, field, new ShortValuesCreator(field,parser)).values;
}
+ @SuppressWarnings("unchecked")
public ShortValues getShorts(IndexReader reader, String field, EntryCreator<ShortValues> creator ) throws IOException
{
return (ShortValues)caches.get(Short.TYPE).get(reader, new Entry(field, creator));
@@ -303,8 +306,8 @@ public class FieldCacheImpl implements F
return getInts(reader, field, new IntValuesCreator( field, parser )).values;
}
- public IntValues getInts(IndexReader reader, String field, EntryCreator<IntValues> creator ) throws IOException
- {
+ @SuppressWarnings("unchecked")
+ public IntValues getInts(IndexReader reader, String field, EntryCreator<IntValues> creator ) throws IOException {
return (IntValues)caches.get(Integer.TYPE).get(reader, new Entry(field, creator));
}
@@ -318,8 +321,8 @@ public class FieldCacheImpl implements F
return getFloats(reader, field, new FloatValuesCreator( field, parser ) ).values;
}
- public FloatValues getFloats(IndexReader reader, String field, EntryCreator<FloatValues> creator ) throws IOException
- {
+ @SuppressWarnings("unchecked")
+ public FloatValues getFloats(IndexReader reader, String field, EntryCreator<FloatValues> creator ) throws IOException {
return (FloatValues)caches.get(Float.TYPE).get(reader, new Entry(field, creator));
}
@@ -332,8 +335,8 @@ public class FieldCacheImpl implements F
return getLongs(reader, field, new LongValuesCreator( field, parser ) ).values;
}
- public LongValues getLongs(IndexReader reader, String field, EntryCreator<LongValues> creator ) throws IOException
- {
+ @SuppressWarnings("unchecked")
+ public LongValues getLongs(IndexReader reader, String field, EntryCreator<LongValues> creator ) throws IOException {
return (LongValues)caches.get(Long.TYPE).get(reader, new Entry(field, creator));
}
@@ -347,38 +350,38 @@ public class FieldCacheImpl implements F
return getDoubles(reader, field, new DoubleValuesCreator( field, parser ) ).values;
}
- public DoubleValues getDoubles(IndexReader reader, String field, EntryCreator<DoubleValues> creator ) throws IOException
- {
+ @SuppressWarnings("unchecked")
+ public DoubleValues getDoubles(IndexReader reader, String field, EntryCreator<DoubleValues> creator ) throws IOException {
return (DoubleValues)caches.get(Double.TYPE).get(reader, new Entry(field, creator));
}
public DocTermsIndex getTermsIndex(IndexReader reader, String field) throws IOException {
- return getTermsIndex(reader, field, new DocTermsIndexCreator<DocTermsIndex>( field ) );
+ return getTermsIndex(reader, field, new DocTermsIndexCreator(field));
}
public DocTermsIndex getTermsIndex(IndexReader reader, String field, boolean fasterButMoreRAM) throws IOException {
- return getTermsIndex(reader, field, new DocTermsIndexCreator<DocTermsIndex>( field,
- fasterButMoreRAM ? DocTermsIndexCreator.FASTER_BUT_MORE_RAM : 0 ) );
+ return getTermsIndex(reader, field, new DocTermsIndexCreator(field,
+ fasterButMoreRAM ? DocTermsIndexCreator.FASTER_BUT_MORE_RAM : 0));
}
- public DocTermsIndex getTermsIndex(IndexReader reader, String field, EntryCreator<DocTermsIndex> creator) throws IOException
- {
+ @SuppressWarnings("unchecked")
+ public DocTermsIndex getTermsIndex(IndexReader reader, String field, EntryCreator<DocTermsIndex> creator) throws IOException {
return (DocTermsIndex)caches.get(DocTermsIndex.class).get(reader, new Entry(field, creator));
}
// TODO: this if DocTermsIndex was already created, we
// should share it...
public DocTerms getTerms(IndexReader reader, String field) throws IOException {
- return getTerms(reader, field, new DocTermsCreator<DocTerms>( field ) );
+ return getTerms(reader, field, new DocTermsCreator(field));
}
public DocTerms getTerms(IndexReader reader, String field, boolean fasterButMoreRAM) throws IOException {
- return getTerms(reader, field, new DocTermsCreator<DocTerms>( field,
- fasterButMoreRAM ? DocTermsCreator.FASTER_BUT_MORE_RAM : 0 ) );
+ return getTerms(reader, field, new DocTermsCreator(field,
+ fasterButMoreRAM ? DocTermsCreator.FASTER_BUT_MORE_RAM : 0));
}
- public DocTerms getTerms(IndexReader reader, String field, EntryCreator<DocTerms> creator) throws IOException
- {
+ @SuppressWarnings("unchecked")
+ public DocTerms getTerms(IndexReader reader, String field, EntryCreator<DocTerms> creator) throws IOException {
return (DocTerms)caches.get(DocTerms.class).get(reader, new Entry(field, creator));
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyQuery.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyQuery.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyQuery.java Wed Oct 20 12:44:28 2010
@@ -20,6 +20,7 @@ package org.apache.lucene.search;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.util.automaton.LevenshteinAutomata;
@@ -135,11 +136,11 @@ public class FuzzyQuery extends MultiTer
}
@Override
- protected TermsEnum getTermsEnum(IndexReader reader) throws IOException {
+ protected TermsEnum getTermsEnum(IndexReader reader, AttributeSource atts) throws IOException {
if (!termLongEnough) { // can only match if it's exact
return new SingleTermsEnum(reader, term);
}
- return new FuzzyTermsEnum(reader, getTerm(), minimumSimilarity, prefixLength);
+ return new FuzzyTermsEnum(reader, atts, getTerm(), minimumSimilarity, prefixLength);
}
/**
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java Wed Oct 20 12:44:28 2010
@@ -22,6 +22,9 @@ import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
@@ -33,7 +36,7 @@ import org.apache.lucene.util.automaton.
import org.apache.lucene.util.automaton.LevenshteinAutomata;
import java.io.IOException;
-import java.util.Arrays;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
@@ -51,7 +54,13 @@ public final class FuzzyTermsEnum extend
private final MultiTermQuery.BoostAttribute boostAtt =
attributes().addAttribute(MultiTermQuery.BoostAttribute.class);
- private float bottom = boostAtt.getMaxNonCompetitiveBoost();
+ private final MultiTermQuery.MaxNonCompetitiveBoostAttribute maxBoostAtt;
+ private final Priv.LevenshteinAutomataAttribute dfaAtt;
+
+ private float bottom;
+ private BytesRef bottomTerm;
+ // nocommit: chicken-and-egg
+ private final Comparator<BytesRef> termComparator = BytesRef.getUTF8SortedAsUnicodeComparator();
private final float minSimilarity;
private final float scale_factor;
@@ -61,8 +70,6 @@ public final class FuzzyTermsEnum extend
private int maxEdits;
private final boolean raw;
- private List<ByteRunAutomaton> runAutomata;
-
private final IndexReader reader;
private final Term term;
private final int termText[];
@@ -77,12 +84,15 @@ public final class FuzzyTermsEnum extend
* valid term if such a term exists.
*
* @param reader Delivers terms.
+ * @param atts {@link AttributeSource} created by the rewrite method of {@link MultiTermQuery}
+ * thats contains information about competitive boosts during rewrite. It is also used
+ * to cache DFAs between segment transitions.
* @param term Pattern term.
* @param minSimilarity Minimum required similarity for terms from the reader.
* @param prefixLength Length of required common prefix. Default value is 0.
* @throws IOException
*/
- public FuzzyTermsEnum(IndexReader reader, Term term,
+ public FuzzyTermsEnum(IndexReader reader, AttributeSource atts, Term term,
final float minSimilarity, final int prefixLength) throws IOException {
if (minSimilarity >= 1.0f && minSimilarity != (int)minSimilarity)
throw new IllegalArgumentException("fractional edit distances are not allowed");
@@ -99,6 +109,7 @@ public final class FuzzyTermsEnum extend
for (int cp, i = 0, j = 0; i < utf16.length(); i += Character.charCount(cp))
termText[j++] = cp = utf16.codePointAt(i);
this.termLength = termText.length;
+ this.dfaAtt = atts.addAttribute(Priv.LevenshteinAutomataAttribute.class);
//The prefix could be longer than the word.
//It's kind of silly though. It means we must match the entire word.
@@ -116,9 +127,10 @@ public final class FuzzyTermsEnum extend
}
this.scale_factor = 1.0f / (1.0f - this.minSimilarity);
- TermsEnum subEnum = getAutomatonEnum(maxEdits, null);
- setEnum(subEnum != null ? subEnum :
- new LinearFuzzyTermsEnum());
+ this.maxBoostAtt = atts.addAttribute(MultiTermQuery.MaxNonCompetitiveBoostAttribute.class);
+ bottom = maxBoostAtt.getMaxNonCompetitiveBoost();
+ bottomTerm = maxBoostAtt.getCompetitiveTerm();
+ bottomChanged(null, true);
}
/**
@@ -127,35 +139,35 @@ public final class FuzzyTermsEnum extend
*/
private TermsEnum getAutomatonEnum(int editDistance, BytesRef lastTerm)
throws IOException {
- initAutomata(editDistance);
- if (runAutomata != null && editDistance < runAutomata.size()) {
+ final List<ByteRunAutomaton> runAutomata = initAutomata(editDistance);
+ if (editDistance < runAutomata.size()) {
return new AutomatonFuzzyTermsEnum(runAutomata.subList(0, editDistance + 1)
- .toArray(new ByteRunAutomaton[0]), lastTerm);
+ .toArray(new ByteRunAutomaton[editDistance + 1]), lastTerm);
} else {
return null;
}
}
/** initialize levenshtein DFAs up to maxDistance, if possible */
- private void initAutomata(int maxDistance) {
- if (runAutomata == null &&
+ private List<ByteRunAutomaton> initAutomata(int maxDistance) {
+ final List<ByteRunAutomaton> runAutomata = dfaAtt.automata();
+ if (runAutomata.size() <= maxDistance &&
maxDistance <= LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE) {
LevenshteinAutomata builder =
new LevenshteinAutomata(UnicodeUtil.newString(termText, realPrefixLength, termText.length - realPrefixLength));
- final ByteRunAutomaton[] ra = new ByteRunAutomaton[maxDistance + 1];
- for (int i = 0; i <= maxDistance; i++) {
+ for (int i = runAutomata.size(); i <= maxDistance; i++) {
Automaton a = builder.toAutomaton(i);
// constant prefix
if (realPrefixLength > 0) {
Automaton prefix = BasicAutomata.makeString(
- UnicodeUtil.newString(termText, 0, realPrefixLength));
+ UnicodeUtil.newString(termText, 0, realPrefixLength));
a = BasicOperations.concatenate(prefix, a);
}
- ra[i] = new ByteRunAutomaton(a);
+ runAutomata.add(new ByteRunAutomaton(a));
}
- runAutomata = Arrays.asList(ra);
}
+ return runAutomata;
}
/** swap in a new actual enum to proxy to */
@@ -169,19 +181,24 @@ public final class FuzzyTermsEnum extend
* fired when the max non-competitive boost has changed. this is the hook to
* swap in a smarter actualEnum
*/
- private void bottomChanged(float boostValue, BytesRef lastTerm)
+ private void bottomChanged(BytesRef lastTerm, boolean init)
throws IOException {
int oldMaxEdits = maxEdits;
+ // true if the last term encountered is lexicographically equal or after the bottom term in the PQ
+ boolean termAfter = bottomTerm == null || (lastTerm != null && termComparator.compare(lastTerm, bottomTerm) >= 0);
+
// as long as the max non-competitive boost is >= the max boost
// for some edit distance, keep dropping the max edit distance.
- while (maxEdits > 0 && boostValue >= calculateMaxBoost(maxEdits))
+ while (maxEdits > 0 && (termAfter ? bottom >= calculateMaxBoost(maxEdits) : bottom > calculateMaxBoost(maxEdits)))
maxEdits--;
- if (oldMaxEdits != maxEdits) { // the maximum n has changed
+ if (oldMaxEdits != maxEdits || init) { // the maximum n has changed
TermsEnum newEnum = getAutomatonEnum(maxEdits, lastTerm);
if (newEnum != null) {
setEnum(newEnum);
+ } else if (init) {
+ setEnum(new LinearFuzzyTermsEnum());
}
}
}
@@ -197,17 +214,26 @@ public final class FuzzyTermsEnum extend
return (similarity - minSimilarity) * scale_factor;
}
+ private BytesRef queuedBottom = null;
+
@Override
public BytesRef next() throws IOException {
+ if (queuedBottom != null) {
+ bottomChanged(queuedBottom, false);
+ queuedBottom = null;
+ }
+
BytesRef term = actualEnum.next();
boostAtt.setBoost(actualBoostAtt.getBoost());
- final float bottom = boostAtt.getMaxNonCompetitiveBoost();
- if (bottom != this.bottom) {
+ final float bottom = maxBoostAtt.getMaxNonCompetitiveBoost();
+ final BytesRef bottomTerm = maxBoostAtt.getCompetitiveTerm();
+ if (term != null && (bottom != this.bottom || bottomTerm != this.bottomTerm)) {
this.bottom = bottom;
+ this.bottomTerm = bottomTerm;
// clone the term before potentially doing something with it
// this is a rare but wonderful occurrence anyway
- bottomChanged(bottom, term == null ? null : (BytesRef) term.clone());
+ queuedBottom = new BytesRef(term);
}
return term;
@@ -524,4 +550,50 @@ public final class FuzzyTermsEnum extend
public float getScaleFactor() {
return scale_factor;
}
+
+ // Wrapper class to hide the attribute from outside!
+ private static final class Priv {
+
+ /** @lucene.internal */
+ public static interface LevenshteinAutomataAttribute extends Attribute {
+ public List<ByteRunAutomaton> automata();
+ }
+
+ /** @lucene.internal */
+ public static final class LevenshteinAutomataAttributeImpl extends AttributeImpl implements LevenshteinAutomataAttribute {
+ private final List<ByteRunAutomaton> automata = new ArrayList<ByteRunAutomaton>();
+
+ public List<ByteRunAutomaton> automata() {
+ return automata;
+ }
+
+ @Override
+ public void clear() {
+ automata.clear();
+ }
+
+ @Override
+ public int hashCode() {
+ return automata.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other)
+ return true;
+ if (!(other instanceof LevenshteinAutomataAttributeImpl))
+ return false;
+ return automata.equals(((LevenshteinAutomataAttributeImpl) other).automata);
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ final List<ByteRunAutomaton> targetAutomata =
+ ((LevenshteinAutomataAttribute) target).automata();
+ targetAutomata.clear();
+ targetAutomata.addAll(automata);
+ }
+ }
+
+ }
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/MultiTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/MultiTermQuery.java?rev=1025539&r1=1025538&r2=1025539&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/MultiTermQuery.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/MultiTermQuery.java Wed Oct 20 12:44:28 2010
@@ -19,19 +19,30 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.io.Serializable;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.PriorityQueue;
+import java.util.Comparator;
+import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.MultiFields;
-import org.apache.lucene.index.Fields;
import org.apache.lucene.index.Terms;
-import org.apache.lucene.queryParser.QueryParser; // for javadoc
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeImpl;
-import org.apache.lucene.util.PagedBytes;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.ByteBlockPool;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefHash;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.ReaderUtil;
+import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray;
/**
* An abstract {@link Query} that matches documents
@@ -39,7 +50,7 @@ import org.apache.lucene.util.PagedBytes
* FilteredTermsEnum} enumeration.
*
* <p>This query cannot be used directly; you must subclass
- * it and define {@link #getTermsEnum} to provide a {@link
+ * it and define {@link #getTermsEnum(IndexReader,AttributeSource)} to provide a {@link
* FilteredTermsEnum} that iterates through the terms to be
* matched.
*
@@ -71,34 +82,25 @@ public abstract class MultiTermQuery ext
protected RewriteMethod rewriteMethod = CONSTANT_SCORE_AUTO_REWRITE_DEFAULT;
transient int numberOfTerms = 0;
- /** Add this {@link Attribute} to a {@link TermsEnum} returned by {@link #getTermsEnum}
+ /** Add this {@link Attribute} to a {@link TermsEnum} returned by {@link #getTermsEnum(IndexReader,AttributeSource)}
* and update the boost on each returned term. This enables to control the boost factor
* for each matching term in {@link #SCORING_BOOLEAN_QUERY_REWRITE} or
* {@link TopTermsBooleanQueryRewrite} mode.
* {@link FuzzyQuery} is using this to take the edit distance into account.
+ * <p><b>Please note:</b> This attribute is intended to be added only by the TermsEnum
+ * to itsself in its constructor and consumed by the {@link RewriteMethod}.
+ * @lucene.internal
*/
public static interface BoostAttribute extends Attribute {
/** Sets the boost in this attribute */
public void setBoost(float boost);
/** Retrieves the boost, default is {@code 1.0f}. */
public float getBoost();
- /** Sets the maximum boost for terms that would never get
- * into the priority queue of {@link MultiTermQuery.TopTermsBooleanQueryRewrite}.
- * This value is not changed by {@link AttributeImpl#clear}
- * and not used in {@code equals()} and {@code hashCode()}.
- * Do not change the value in the {@link TermsEnum}!
- */
- public void setMaxNonCompetitiveBoost(float maxNonCompetitiveBoost);
- /** Retrieves the maximum boost that is not competitive,
- * default is megative infinity. You can use this boost value
- * as a hint when writing the {@link TermsEnum}.
- */
- public float getMaxNonCompetitiveBoost();
}
/** Implementation class for {@link BoostAttribute}. */
public static final class BoostAttributeImpl extends AttributeImpl implements BoostAttribute {
- private float boost = 1.0f, maxNonCompetitiveBoost = Float.NEGATIVE_INFINITY;
+ private float boost = 1.0f;
public void setBoost(float boost) {
this.boost = boost;
@@ -107,8 +109,61 @@ public abstract class MultiTermQuery ext
public float getBoost() {
return boost;
}
+
+ @Override
+ public void clear() {
+ boost = 1.0f;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other)
+ return true;
+ if (other instanceof BoostAttributeImpl)
+ return ((BoostAttributeImpl) other).boost == boost;
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Float.floatToIntBits(boost);
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ ((BoostAttribute) target).setBoost(boost);
+ }
+ }
+
+ /** Add this {@link Attribute} to a fresh {@link AttributeSource} before calling
+ * {@link #getTermsEnum(IndexReader,AttributeSource)}.
+ * {@link FuzzyQuery} is using this to control its internal behaviour
+ * to only return competitive terms.
+ * <p><b>Please note:</b> This attribute is intended to be added by the {@link RewriteMethod}
+ * to an empty {@link AttributeSource} that is shared for all segments
+ * during query rewrite. This attribute source is passed to all segment enums
+ * on {@link #getTermsEnum(IndexReader,AttributeSource)}.
+ * {@link TopTermsBooleanQueryRewrite} uses this attribute to
+ * inform all enums about the current boost, that is not competitive.
+ * @lucene.internal
+ */
+ public static interface MaxNonCompetitiveBoostAttribute extends Attribute {
+ /** This is the maximum boost that would not be competitive. */
+ public void setMaxNonCompetitiveBoost(float maxNonCompetitiveBoost);
+ /** This is the maximum boost that would not be competitive. Default is negative infinity, which means every term is competitive. */
+ public float getMaxNonCompetitiveBoost();
+ /** This is the term or <code>null<code> of the term that triggered the boost change. */
+ public void setCompetitiveTerm(BytesRef competitiveTerm);
+ /** This is the term or <code>null<code> of the term that triggered the boost change. Default is <code>null</code>, which means every term is competitoive. */
+ public BytesRef getCompetitiveTerm();
+ }
+
+ /** Implementation class for {@link MaxNonCompetitiveBoostAttribute}. */
+ public static final class MaxNonCompetitiveBoostAttributeImpl extends AttributeImpl implements MaxNonCompetitiveBoostAttribute {
+ private float maxNonCompetitiveBoost = Float.NEGATIVE_INFINITY;
+ private BytesRef competitiveTerm = null;
- public void setMaxNonCompetitiveBoost(float maxNonCompetitiveBoost) {
+ public void setMaxNonCompetitiveBoost(final float maxNonCompetitiveBoost) {
this.maxNonCompetitiveBoost = maxNonCompetitiveBoost;
}
@@ -116,28 +171,44 @@ public abstract class MultiTermQuery ext
return maxNonCompetitiveBoost;
}
+ public void setCompetitiveTerm(final BytesRef competitiveTerm) {
+ this.competitiveTerm = competitiveTerm;
+ }
+
+ public BytesRef getCompetitiveTerm() {
+ return competitiveTerm;
+ }
+
@Override
public void clear() {
- boost = 1.0f;
+ maxNonCompetitiveBoost = Float.NEGATIVE_INFINITY;
+ competitiveTerm = null;
}
@Override
public boolean equals(Object other) {
if (this == other)
return true;
- if (other instanceof BoostAttributeImpl)
- return ((BoostAttributeImpl) other).boost == boost;
+ if (other instanceof MaxNonCompetitiveBoostAttributeImpl) {
+ final MaxNonCompetitiveBoostAttributeImpl o = (MaxNonCompetitiveBoostAttributeImpl) other;
+ return (o.maxNonCompetitiveBoost == maxNonCompetitiveBoost)
+ && (o.competitiveTerm == null ? competitiveTerm == null : o.competitiveTerm.equals(competitiveTerm));
+ }
return false;
}
@Override
public int hashCode() {
- return Float.floatToIntBits(boost);
+ int hash = Float.floatToIntBits(maxNonCompetitiveBoost);
+ if (competitiveTerm != null) hash = 31 * hash + competitiveTerm.hashCode();
+ return hash;
}
@Override
public void copyTo(AttributeImpl target) {
- ((BoostAttribute) target).setBoost(boost);
+ final MaxNonCompetitiveBoostAttributeImpl t = (MaxNonCompetitiveBoostAttributeImpl) target;
+ t.setMaxNonCompetitiveBoost(maxNonCompetitiveBoost);
+ t.setCompetitiveTerm(competitiveTerm);
}
}
@@ -177,69 +248,85 @@ public abstract class MultiTermQuery ext
private abstract static class BooleanQueryRewrite extends RewriteMethod {
protected final int collectTerms(IndexReader reader, MultiTermQuery query, TermCollector collector) throws IOException {
- final Fields fields = MultiFields.getFields(reader);
- if (fields == null) {
- // reader has no fields
- return 0;
- }
-
- final Terms terms = fields.terms(query.field);
- if (terms == null) {
- // field does not exist
- return 0;
- }
-
- final TermsEnum termsEnum = query.getTermsEnum(reader);
- assert termsEnum != null;
-
- if (termsEnum == TermsEnum.EMPTY)
- return 0;
- final BoostAttribute boostAtt =
- termsEnum.attributes().addAttribute(BoostAttribute.class);
- collector.boostAtt = boostAtt;
+ final List<IndexReader> subReaders = new ArrayList<IndexReader>();
+ ReaderUtil.gatherSubReaders(subReaders, reader);
int count = 0;
- BytesRef bytes;
- while ((bytes = termsEnum.next()) != null) {
- if (collector.collect(termsEnum, bytes, boostAtt.getBoost())) {
- termsEnum.cacheCurrentTerm();
- count++;
- } else {
- break;
+ Comparator<BytesRef> lastTermComp = null;
+
+ for (IndexReader r : subReaders) {
+ final Fields fields = r.fields();
+ if (fields == null) {
+ // reader has no fields
+ continue;
+ }
+
+ final Terms terms = fields.terms(query.field);
+ if (terms == null) {
+ // field does not exist
+ continue;
+ }
+
+ final TermsEnum termsEnum = query.getTermsEnum(r, collector.attributes);
+ assert termsEnum != null;
+
+ if (termsEnum == TermsEnum.EMPTY)
+ continue;
+
+ // Check comparator compatibility:
+ final Comparator<BytesRef> newTermComp = termsEnum.getComparator();
+ if (lastTermComp != null && newTermComp != lastTermComp)
+ throw new RuntimeException("term comparator should not change between segments: "+lastTermComp+" != "+newTermComp);
+ lastTermComp = newTermComp;
+
+ collector.setNextEnum(termsEnum);
+ BytesRef bytes;
+ while ((bytes = termsEnum.next()) != null) {
+ if (collector.collect(bytes)) {
+ termsEnum.cacheCurrentTerm();
+ count++;
+ } else {
+ return count; // interrupt whole term collection, so also don't iterate other subReaders
+ }
}
}
- collector.boostAtt = null;
return count;
}
protected static abstract class TermCollector {
- private BoostAttribute boostAtt = null;
+ /** attributes used for communication with the enum */
+ public final AttributeSource attributes = new AttributeSource();
/** return false to stop collecting */
- public abstract boolean collect(TermsEnum termsEnum, BytesRef bytes, float boost) throws IOException;
+ public abstract boolean collect(BytesRef bytes) throws IOException;
- /** set the minimum boost as a hint for the term producer */
- protected final void setMaxNonCompetitiveBoost(float maxNonCompetitiveBoost) {
- assert boostAtt != null;
- boostAtt.setMaxNonCompetitiveBoost(maxNonCompetitiveBoost);
- }
+ /** the next segment's {@link TermsEnum} that is used to collect terms */
+ public abstract void setNextEnum(TermsEnum termsEnum) throws IOException;
}
}
private static class ScoringBooleanQueryRewrite extends BooleanQueryRewrite {
@Override
public Query rewrite(final IndexReader reader, final MultiTermQuery query) throws IOException {
- final BooleanQuery result = new BooleanQuery(true);
+ final ParallelArraysTermCollector col = new ParallelArraysTermCollector();
+ collectTerms(reader, query, col);
+
final Term placeholderTerm = new Term(query.field);
- query.incTotalNumberOfTerms(collectTerms(reader, query, new TermCollector() {
- @Override
- public boolean collect(TermsEnum termsEnum, BytesRef bytes, float boost) {
- // add new TQ, we must clone the term, else it may get overwritten!
- TermQuery tq = new TermQuery(placeholderTerm.createTerm(new BytesRef(bytes)), termsEnum.docFreq());
- tq.setBoost(query.getBoost() * boost); // set the boost
- result.add(tq, BooleanClause.Occur.SHOULD); // add to query
- return true;
+ final BooleanQuery result = new BooleanQuery(true);
+ final int size = col.terms.size();
+ if (size > 0) {
+ final int sort[] = col.terms.sort(col.termsEnum.getComparator());
+ final int[] docFreq = col.array.docFreq;
+ final float[] boost = col.array.boost;
+ for (int i = 0; i < size; i++) {
+ final int pos = sort[i];
+ final Term term = placeholderTerm.createTerm(col.terms.get(pos, new BytesRef()));
+ assert reader.docFreq(term) == docFreq[pos];
+ final TermQuery tq = new TermQuery(term, docFreq[pos]);
+ tq.setBoost(query.getBoost() * boost[pos]);
+ result.add(tq, BooleanClause.Occur.SHOULD);
}
- }));
+ }
+ query.incTotalNumberOfTerms(size);
return result;
}
@@ -247,6 +334,75 @@ public abstract class MultiTermQuery ext
protected Object readResolve() {
return SCORING_BOOLEAN_QUERY_REWRITE;
}
+
+ static final class ParallelArraysTermCollector extends TermCollector {
+ final TermFreqBoostByteStart array = new TermFreqBoostByteStart(16);
+ final BytesRefHash terms = new BytesRefHash(new ByteBlockPool(new ByteBlockPool.DirectAllocator()), 16, array);
+ TermsEnum termsEnum;
+
+ private BoostAttribute boostAtt;
+
+ @Override
+ public void setNextEnum(TermsEnum termsEnum) throws IOException {
+ this.termsEnum = termsEnum;
+ this.boostAtt = termsEnum.attributes().addAttribute(BoostAttribute.class);
+ }
+
+ @Override
+ public boolean collect(BytesRef bytes) {
+ final int e = terms.add(bytes);
+ if (e < 0 ) {
+ // duplicate term: update docFreq
+ final int pos = (-e)-1;
+ array.docFreq[pos] += termsEnum.docFreq();
+ assert array.boost[pos] == boostAtt.getBoost() : "boost should be equal in all segment TermsEnums";
+ } else {
+ // new entry: we populate the entry initially
+ array.docFreq[e] = termsEnum.docFreq();
+ array.boost[e] = boostAtt.getBoost();
+ }
+ // if the new entry reaches the max clause count, we exit early
+ if (e >= BooleanQuery.getMaxClauseCount())
+ throw new BooleanQuery.TooManyClauses();
+ return true;
+ }
+ }
+
+ /** Special implementation of BytesStartArray that keeps parallel arrays for boost and docFreq */
+ static final class TermFreqBoostByteStart extends DirectBytesStartArray {
+ int[] docFreq;
+ float[] boost;
+
+ public TermFreqBoostByteStart(int initSize) {
+ super(initSize);
+ }
+
+ @Override
+ public int[] init() {
+ final int[] ord = super.init();
+ boost = new float[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_FLOAT)];
+ docFreq = new int[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_INT)];
+ assert boost.length >= ord.length && docFreq.length >= ord.length;
+ return ord;
+ }
+
+ @Override
+ public int[] grow() {
+ final int[] ord = super.grow();
+ docFreq = ArrayUtil.grow(docFreq, ord.length);
+ boost = ArrayUtil.grow(boost, ord.length);
+ assert boost.length >= ord.length && docFreq.length >= ord.length;
+ return ord;
+ }
+
+ @Override
+ public int[] clear() {
+ boost = null;
+ docFreq = null;
+ return super.clear();
+ }
+
+ }
}
/** A rewrite method that first translates each term into
@@ -291,44 +447,92 @@ public abstract class MultiTermQuery ext
final int maxSize = Math.min(size, BooleanQuery.getMaxClauseCount());
final PriorityQueue<ScoreTerm> stQueue = new PriorityQueue<ScoreTerm>();
collectTerms(reader, query, new TermCollector() {
+ private final MaxNonCompetitiveBoostAttribute maxBoostAtt =
+ attributes.addAttribute(MaxNonCompetitiveBoostAttribute.class);
+
+ private final Map<BytesRef,ScoreTerm> visitedTerms = new HashMap<BytesRef,ScoreTerm>();
+
+ private TermsEnum termsEnum;
+ private Comparator<BytesRef> termComp;
+ private BoostAttribute boostAtt;
+ private ScoreTerm st;
+
@Override
- public boolean collect(TermsEnum termsEnum, BytesRef bytes, float boost) {
+ public void setNextEnum(TermsEnum termsEnum) throws IOException {
+ this.termsEnum = termsEnum;
+ this.termComp = termsEnum.getComparator();
+ // lazy init the initial ScoreTerm because comparator is not known on ctor:
+ if (st == null)
+ st = new ScoreTerm(this.termComp);
+ boostAtt = termsEnum.attributes().addAttribute(BoostAttribute.class);
+ }
+
+ @Override
+ public boolean collect(BytesRef bytes) {
+ final float boost = boostAtt.getBoost();
// ignore uncompetetive hits
- if (stQueue.size() >= maxSize && boost <= stQueue.peek().boost)
- return true;
- // add new entry in PQ, we must clone the term, else it may get overwritten!
- st.bytes.copy(bytes);
- st.boost = boost;
- st.docFreq = termsEnum.docFreq();
- stQueue.offer(st);
- // possibly drop entries from queue
- st = (stQueue.size() > maxSize) ? stQueue.poll() : new ScoreTerm();
- setMaxNonCompetitiveBoost((stQueue.size() >= maxSize) ? stQueue.peek().boost : Float.NEGATIVE_INFINITY);
+ if (stQueue.size() == maxSize) {
+ final ScoreTerm t = stQueue.peek();
+ if (boost < t.boost)
+ return true;
+ if (boost == t.boost && termComp.compare(bytes, t.bytes) > 0)
+ return true;
+ }
+ ScoreTerm t = visitedTerms.get(bytes);
+ if (t != null) {
+ // if the term is already in the PQ, only update docFreq of term in PQ
+ t.docFreq += termsEnum.docFreq();
+ assert t.boost == boost : "boost should be equal in all segment TermsEnums";
+ } else {
+ // add new entry in PQ, we must clone the term, else it may get overwritten!
+ st.bytes.copy(bytes);
+ st.boost = boost;
+ st.docFreq = termsEnum.docFreq();
+ visitedTerms.put(st.bytes, st);
+ stQueue.offer(st);
+ // possibly drop entries from queue
+ if (stQueue.size() > maxSize) {
+ st = stQueue.poll();
+ visitedTerms.remove(st.bytes);
+ } else {
+ st = new ScoreTerm(termComp);
+ }
+ assert stQueue.size() <= maxSize : "the PQ size must be limited to maxSize";
+ // set maxBoostAtt with values to help FuzzyTermsEnum to optimize
+ if (stQueue.size() == maxSize) {
+ t = stQueue.peek();
+ maxBoostAtt.setMaxNonCompetitiveBoost(t.boost);
+ maxBoostAtt.setCompetitiveTerm(t.bytes);
+ }
+ }
return true;
}
-
- // reusable instance
- private ScoreTerm st = new ScoreTerm();
});
final Term placeholderTerm = new Term(query.field);
final BooleanQuery bq = new BooleanQuery(true);
- for (final ScoreTerm st : stQueue) {
- // add new query, we must clone the term, else it may get overwritten!
- Query tq = getQuery(placeholderTerm.createTerm(st.bytes), st.docFreq);
+ final ScoreTerm[] scoreTerms = stQueue.toArray(new ScoreTerm[stQueue.size()]);
+ Arrays.sort(scoreTerms, new Comparator<ScoreTerm>() {
+ public int compare(ScoreTerm st1, ScoreTerm st2) {
+ assert st1.termComp == st2.termComp :
+ "term comparator should not change between segments";
+ return st1.termComp.compare(st1.bytes, st2.bytes);
+ }
+ });
+ for (final ScoreTerm st : scoreTerms) {
+ final Term term = placeholderTerm.createTerm(st.bytes);
+ assert reader.docFreq(term) == st.docFreq;
+ Query tq = getQuery(term, st.docFreq);
tq.setBoost(query.getBoost() * st.boost); // set the boost
bq.add(tq, BooleanClause.Occur.SHOULD); // add to query
}
- query.incTotalNumberOfTerms(bq.clauses().size());
+ query.incTotalNumberOfTerms(scoreTerms.length);
return bq;
}
@Override
public int hashCode() {
- final int prime = 17;
- int result = 1;
- result = prime * result + size;
- return result;
+ return 31 * size;
}
@Override
@@ -341,15 +545,20 @@ public abstract class MultiTermQuery ext
return true;
}
- private static class ScoreTerm implements Comparable<ScoreTerm> {
+ static final class ScoreTerm implements Comparable<ScoreTerm> {
+ public final Comparator<BytesRef> termComp;
+
public final BytesRef bytes = new BytesRef();
public float boost;
public int docFreq;
+ public ScoreTerm(Comparator<BytesRef> termComp) {
+ this.termComp = termComp;
+ }
+
public int compareTo(ScoreTerm other) {
if (this.boost == other.boost)
- // TODO: is it OK to use default compare here?
- return other.bytes.compareTo(this.bytes);
+ return termComp.compare(other.bytes, this.bytes);
else
return Float.compare(this.boost, other.boost);
}
@@ -362,8 +571,8 @@ public abstract class MultiTermQuery ext
* scores as computed by the query.
*
* <p>
- * This rewrite mode only uses the top scoring terms so it will not overflow
- * the boolean max clause count. It is the default rewrite mode for
+ * This rewrite method only uses the top scoring terms so it will not overflow
+ * the boolean max clause count. It is the default rewrite method for
* {@link FuzzyQuery}.
*
* @see #setRewriteMethod
@@ -510,63 +719,61 @@ public abstract class MultiTermQuery ext
final int docCountCutoff = (int) ((docCountPercent / 100.) * reader.maxDoc());
final int termCountLimit = Math.min(BooleanQuery.getMaxClauseCount(), termCountCutoff);
- final CutOffTermCollector col = new CutOffTermCollector(reader, query.field, docCountCutoff, termCountLimit);
+ final CutOffTermCollector col = new CutOffTermCollector(docCountCutoff, termCountLimit);
collectTerms(reader, query, col);
-
+ final int size = col.pendingTerms.size();
if (col.hasCutOff) {
return CONSTANT_SCORE_FILTER_REWRITE.rewrite(reader, query);
- } else if (col.termCount == 0) {
+ } else if (size == 0) {
return new BooleanQuery(true);
} else {
- final PagedBytes.Reader bytesReader = col.pendingTerms.freeze(false);
- try {
- final BooleanQuery bq = new BooleanQuery(true);
- final Term placeholderTerm = new Term(query.field);
- long start = col.startOffset;
- for(int i = 0; i < col.termCount; i++) {
- final BytesRef bytes = new BytesRef();
- start = bytesReader.fillUsingLengthPrefix3(bytes, start);
- bq.add(new TermQuery(placeholderTerm.createTerm(bytes)), BooleanClause.Occur.SHOULD);
- }
- // Strip scores
- final Query result = new ConstantScoreQuery(new QueryWrapperFilter(bq));
- result.setBoost(query.getBoost());
- query.incTotalNumberOfTerms(col.termCount);
- return result;
- } finally {
- bytesReader.close();
+ final BooleanQuery bq = new BooleanQuery(true);
+ final Term placeholderTerm = new Term(query.field);
+ final BytesRefHash pendingTerms = col.pendingTerms;
+ final int sort[] = pendingTerms.sort(col.termsEnum.getComparator());
+ for(int i = 0; i < size; i++) {
+ // docFreq is not used for constant score here, we pass 1
+ // to explicitely set a fake value, so it's not calculated
+ bq.add(new TermQuery(
+ placeholderTerm.createTerm(pendingTerms.get(sort[i], new BytesRef())), 1
+ ), BooleanClause.Occur.SHOULD);
}
+ // Strip scores
+ final Query result = new ConstantScoreQuery(new QueryWrapperFilter(bq));
+ result.setBoost(query.getBoost());
+ query.incTotalNumberOfTerms(size);
+ return result;
}
}
- private static final class CutOffTermCollector extends TermCollector {
- CutOffTermCollector(IndexReader reader, String field, int docCountCutoff, int termCountLimit) {
- this.reader = reader;
- this.field = field;
+ static final class CutOffTermCollector extends TermCollector {
+ CutOffTermCollector(int docCountCutoff, int termCountLimit) {
this.docCountCutoff = docCountCutoff;
this.termCountLimit = termCountLimit;
}
- public boolean collect(TermsEnum termsEnum, BytesRef bytes, float boost) throws IOException {
- termCount++;
- if (termCount >= termCountLimit || docVisitCount >= docCountCutoff) {
+ @Override
+ public void setNextEnum(TermsEnum termsEnum) throws IOException {
+ this.termsEnum = termsEnum;
+ }
+
+ @Override
+ public boolean collect(BytesRef bytes) throws IOException {
+ if (pendingTerms.size() >= termCountLimit || docVisitCount >= docCountCutoff) {
hasCutOff = true;
return false;
}
- pendingTerms.copyUsingLengthPrefix(bytes);
+ pendingTerms.add(bytes);
docVisitCount += termsEnum.docFreq();
return true;
}
int docVisitCount = 0;
boolean hasCutOff = false;
- int termCount = 0;
-
- final IndexReader reader;
- final String field;
+ TermsEnum termsEnum;
+
final int docCountCutoff, termCountLimit;
- final PagedBytes pendingTerms = new PagedBytes(15); // max term size is 32 KiB
- final long startOffset = pendingTerms.getPointer();
+ final BytesRefHash pendingTerms = new BytesRefHash();
}
@Override
@@ -644,8 +851,20 @@ public abstract class MultiTermQuery ext
* field does exist). This method should not return null
* (should instead return {@link TermsEnum#EMPTY} if no
* terms match). The TermsEnum must already be
- * positioned to the first matching term. */
- protected abstract TermsEnum getTermsEnum(IndexReader reader) throws IOException;
+ * positioned to the first matching term.
+ * The given {@link AttributeSource} is passed by the {@link RewriteMethod} to
+ * provide attributes, the rewrite method uses to inform about e.g. maximum competitive boosts.
+ * This is currently only used by {@link TopTermsBooleanQueryRewrite}
+ */
+ protected abstract TermsEnum getTermsEnum(IndexReader reader, AttributeSource atts) throws IOException;
+
+ /** Convenience method, if no attributes are needed:
+ * This simply passes empty attributes and is equal to:
+ * <code>getTermsEnum(reader, new AttributeSource())</code>
+ */
+ protected final TermsEnum getTermsEnum(IndexReader reader) throws IOException {
+ return getTermsEnum(reader, new AttributeSource());
+ }
/**
* Expert: Return the number of unique terms visited during execution of the query.
Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct 20 12:44:28 2010
@@ -1,5 +1,6 @@
/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512
/lucene/dev/branches/preflexfixes/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:967125-979432
+/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:1021635-1024556,1025532-1025536
/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:824912-931101
/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:909334,948516
/lucene/java/trunk/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:924483-924731,924781,925176-925462