You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/12/22 11:27:37 UTC
svn commit: r1721331 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/suggest/
lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/
lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/ solr/
solr/core/ solr/core/src/...
Author: mikemccand
Date: Tue Dec 22 10:27:36 2015
New Revision: 1721331
URL: http://svn.apache.org/viewvc?rev=1721331&view=rev
Log:
LUCENE-6939: add exponential reciprocal scoring mode to BlendedInfixSuggester
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java
lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/spelling/suggest/fst/BlendedInfixLookupFactory.java
lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/spelling/suggest/TestBlendedInfixSuggestions.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1721331&r1=1721330&r2=1721331&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Tue Dec 22 10:27:36 2015
@@ -12,6 +12,10 @@ New Features
for LONG and INT fields with NUMERIC and SORTED_NUMERIC doc values.
(Alexey Zelin via Mikhail Khludnev)
+* LUCENE-6939: Add exponential reciprocal scoring to
+ BlendedInfixSuggester, to even more strongly favor suggestions that
+ match closer to the beginning (Arcadius Ahouansou via Mike McCandless)
+
API Changes
* LUCENE-6908: GeoUtils static relational methods have been refactored to new
Modified: lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java?rev=1721331&r1=1721330&r2=1721331&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java (original)
+++ lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java Tue Dec 22 10:27:36 2015
@@ -63,6 +63,8 @@ public class BlendedInfixSuggester exten
*/
protected static double LINEAR_COEF = 0.10;
+ private Double exponent = 2.0;
+
/**
* Default factor
*/
@@ -89,6 +91,8 @@ public class BlendedInfixSuggester exten
POSITION_LINEAR,
/** weight/(1+position) */
POSITION_RECIPROCAL,
+ /** weight/pow(1+position, exponent) */
+ POSITION_EXPONENTIAL_RECIPROCAL
// TODO:
//SCORE
}
@@ -136,13 +140,14 @@ public class BlendedInfixSuggester exten
this.blenderType = blenderType;
this.numFactor = numFactor;
}
-
+
/**
* Create a new instance, loading from a previously built
* directory, if it exists.
*
* @param blenderType Type of blending strategy, see BlenderType for more precisions
* @param numFactor Factor to multiply the number of searched elements before ponderate
+ * @param exponent exponent used only when blenderType is BlenderType.POSITION_EXPONENTIAL_RECIPROCAL
* @param commitOnBuild Call commit after the index has finished building. This would persist the
* suggester index to disk and future instances of this suggester can use this pre-built dictionary.
* @param allTermsRequired All terms in the suggest query must be matched.
@@ -150,13 +155,16 @@ public class BlendedInfixSuggester exten
* @throws IOException If there are problems opening the underlying Lucene index.
*/
public BlendedInfixSuggester(Directory dir, Analyzer indexAnalyzer, Analyzer queryAnalyzer,
- int minPrefixChars, BlenderType blenderType, int numFactor,
+ int minPrefixChars, BlenderType blenderType, int numFactor, Double exponent,
boolean commitOnBuild, boolean allTermsRequired, boolean highlight) throws IOException {
super(dir, indexAnalyzer, queryAnalyzer, minPrefixChars, commitOnBuild, allTermsRequired, highlight);
this.blenderType = blenderType;
this.numFactor = numFactor;
+ if(exponent != null) {
+ this.exponent = exponent;
+ }
}
-
+
@Override
public List<Lookup.LookupResult> lookup(CharSequence key, Set<BytesRef> contexts, boolean onlyMorePopular, int num) throws IOException {
// Don't * numFactor here since we do it down below, once, in the call chain:
@@ -321,6 +329,10 @@ public class BlendedInfixSuggester exten
coefficient = 1. / (position + 1);
break;
+ case POSITION_EXPONENTIAL_RECIPROCAL:
+ coefficient = 1. / Math.pow((position + 1.0), exponent);
+ break;
+
default:
coefficient = 1;
}
Modified: lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java?rev=1721331&r1=1721330&r2=1721331&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java (original)
+++ lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java Tue Dec 22 10:27:36 2015
@@ -112,6 +112,18 @@ public class BlendedInfixSuggesterTest e
assertEquals(w, getInResults(suggester, "top", pl, 1));
assertEquals((int) (w * 1 / (1 + 2)), getInResults(suggester, "the", pl, 1));
assertEquals((int) (w * 1 / (1 + 3)), getInResults(suggester, "lake", pl, 1));
+ suggester.close();
+
+ // BlenderType.EXPONENTIAL_RECIPROCAL is using 1/(pow(1+p, exponent)) * w where w is weight and p the position of the word
+ suggester = new BlendedInfixSuggester(newFSDirectory(tempDir), a, a,
+ AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS,
+ BlendedInfixSuggester.BlenderType.POSITION_EXPONENTIAL_RECIPROCAL, 1, 4.0, false, true, false);
+
+ suggester.build(new InputArrayIterator(keys));
+
+ assertEquals(w, getInResults(suggester, "top", pl, 1));
+ assertEquals((int) (w * 1 / (Math.pow(1 + 2, 4.0))), getInResults(suggester, "the", pl, 1));
+ assertEquals((int) (w * 1 / (Math.pow(1 + 3, 4.0))), getInResults(suggester, "lake", pl, 1));
suggester.close();
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/spelling/suggest/fst/BlendedInfixLookupFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/spelling/suggest/fst/BlendedInfixLookupFactory.java?rev=1721331&r1=1721330&r2=1721331&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/spelling/suggest/fst/BlendedInfixLookupFactory.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/spelling/suggest/fst/BlendedInfixLookupFactory.java Tue Dec 22 10:27:36 2015
@@ -21,13 +21,14 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Locale;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester;
-import org.apache.lucene.search.suggest.analyzing.BlendedInfixSuggester;
import org.apache.lucene.search.suggest.analyzing.BlendedInfixSuggester.BlenderType;
+import org.apache.lucene.search.suggest.analyzing.BlendedInfixSuggester;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.util.NamedList;
@@ -49,7 +50,9 @@ public class BlendedInfixLookupFactory e
* reciprocal: weight/(1+position)
*/
private static final String BLENDER_TYPE = "blenderType";
-
+
+ private static final String EXPONENT = "exponent";
+
/**
* Factor to multiply the number of searched elements
* Default is 10
@@ -107,11 +110,13 @@ public class BlendedInfixLookupFactory e
int numFactor = params.get(NUM_FACTOR) != null
? Integer.parseInt(params.get(NUM_FACTOR).toString())
: BlendedInfixSuggester.DEFAULT_NUM_FACTOR;
-
+
+ Double exponent = params.get(EXPONENT) == null ? null : Double.valueOf(params.get(EXPONENT).toString());
+
try {
return new BlendedInfixSuggester(FSDirectory.open(new File(indexPath).toPath()),
indexAnalyzer, queryAnalyzer, minPrefixChars,
- blenderType, numFactor, true,
+ blenderType, numFactor, exponent, true,
allTermsRequired, highlight) {
@Override
public List<LookupResult> lookup(CharSequence key, Set<BytesRef> contexts, int num, boolean allTermsRequired, boolean doHighlight) throws IOException {
@@ -144,10 +149,8 @@ public class BlendedInfixLookupFactory e
private BlenderType getBlenderType(Object blenderTypeParam) {
BlenderType blenderType = BlenderType.POSITION_LINEAR;
if (blenderTypeParam != null) {
- String blenderTypeStr = blenderTypeParam.toString();
- if (blenderTypeStr.equalsIgnoreCase("reciprocal")) {
- blenderType = BlenderType.POSITION_RECIPROCAL;
- }
+ String blenderTypeStr = blenderTypeParam.toString().toUpperCase(Locale.ROOT);
+ blenderType = BlenderType.valueOf(blenderTypeStr);
}
return blenderType;
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml?rev=1721331&r1=1721330&r2=1721331&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml Tue Dec 22 10:27:36 2015
@@ -165,7 +165,8 @@
<str name="indexPath">blendedInfixDir1</str>
<!-- Suggester properties -->
- <str name="blenderType">linear</str>
+ <!--full enum name of BlenderType-->
+ <str name="blenderType">position_linear</str>
<str name="suggestAnalyzerFieldType">text</str>
</lst>
@@ -178,10 +179,40 @@
<str name="indexPath">blendedInfixDir2</str>
<!-- Suggester properties -->
- <str name="blenderType">reciprocal</str>
+ <!--full enum name of BlenderType-->
+ <str name="blenderType">position_reciprocal</str>
<str name="suggestAnalyzerFieldType">text</str>
</lst>
-
+
+ <lst name="suggester">
+ <!-- exponential with default exponent -->
+ <str name="name">blended_infix_suggest_exponential_reciprocal</str>
+ <str name="lookupImpl">BlendedInfixLookupFactory</str>
+ <str name="dictionaryImpl">FileDictionaryFactory</str>
+ <str name="buildOnCommit">false</str>
+ <str name="sourceLocation">blendedInfixSuggest.txt</str>
+ <str name="indexPath">blended_infix_suggest_exponential_reciprocal</str>
+
+ <!-- Suggester properties -->
+ <!--full enum name of BlenderType-->
+ <str name="blenderType">position_exponential_reciprocal</str>
+ <str name="suggestAnalyzerFieldType">text</str>
+ </lst>
+
+ <lst name="suggester">
+ <str name="name">blended_infix_suggest_exponential_reciprocal_1</str>
+ <str name="lookupImpl">BlendedInfixLookupFactory</str>
+ <str name="dictionaryImpl">FileDictionaryFactory</str>
+ <str name="buildOnCommit">false</str>
+ <str name="sourceLocation">blendedInfixSuggest.txt</str>
+ <str name="indexPath">blended_infix_suggest_exponential_reciprocal_1</str>
+
+ <!-- Suggester properties -->
+ <str name="blenderType">position_exponential_reciprocal</str>
+ <double name="exponent">1.0</double>
+ <str name="suggestAnalyzerFieldType">text</str>
+ </lst>
+
</searchComponent>
<!-- FuzzyLookup suggest component with FileDictionaryFactory -->
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/spelling/suggest/TestBlendedInfixSuggestions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/spelling/suggest/TestBlendedInfixSuggestions.java?rev=1721331&r1=1721330&r2=1721331&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/spelling/suggest/TestBlendedInfixSuggestions.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/spelling/suggest/TestBlendedInfixSuggestions.java Tue Dec 22 10:27:36 2015
@@ -58,7 +58,37 @@ public class TestBlendedInfixSuggestions
"//lst[@name='suggest']/lst[@name='blended_infix_suggest_reciprocal']/lst[@name='the']/arr[@name='suggestions']/lst[3]/str[@name='payload'][.='star']"
);
}
-
+
+ public void testExponentialReciprocalBlenderTypeExponent1() { //exponent=1 will give same output as reciprocal
+ assertQ(req("qt", URI, "q", "the", SuggesterParams.SUGGEST_COUNT, "10", SuggesterParams.SUGGEST_DICT, "blended_infix_suggest_exponential_reciprocal_1"),
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal_1']/lst[@name='the']/int[@name='numFound'][.='3']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal_1']/lst[@name='the']/arr[@name='suggestions']/lst[1]/str[@name='term'][.='<b>the</b> returned']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal_1']/lst[@name='the']/arr[@name='suggestions']/lst[1]/long[@name='weight'][.='10']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal_1']/lst[@name='the']/arr[@name='suggestions']/lst[1]/str[@name='payload'][.='ret']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal_1']/lst[@name='the']/arr[@name='suggestions']/lst[2]/str[@name='term'][.='top of <b>the</b> lake']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal_1']/lst[@name='the']/arr[@name='suggestions']/lst[2]/long[@name='weight'][.='6']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal_1']/lst[@name='the']/arr[@name='suggestions']/lst[2]/str[@name='payload'][.='lake']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal_1']/lst[@name='the']/arr[@name='suggestions']/lst[3]/str[@name='term'][.='star wars: episode v - <b>the</b> empire strikes back']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal_1']/lst[@name='the']/arr[@name='suggestions']/lst[3]/long[@name='weight'][.='2']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal_1']/lst[@name='the']/arr[@name='suggestions']/lst[3]/str[@name='payload'][.='star']"
+ );
+ }
+
+ public void testExponentialReciprocalBlenderType() { // default is exponent=2.0
+ assertQ(req("qt", URI, "q", "the", SuggesterParams.SUGGEST_COUNT, "10", SuggesterParams.SUGGEST_DICT, "blended_infix_suggest_exponential_reciprocal"),
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal']/lst[@name='the']/int[@name='numFound'][.='3']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal']/lst[@name='the']/arr[@name='suggestions']/lst[1]/str[@name='term'][.='<b>the</b> returned']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal']/lst[@name='the']/arr[@name='suggestions']/lst[1]/long[@name='weight'][.='10']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal']/lst[@name='the']/arr[@name='suggestions']/lst[1]/str[@name='payload'][.='ret']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal']/lst[@name='the']/arr[@name='suggestions']/lst[2]/str[@name='term'][.='top of <b>the</b> lake']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal']/lst[@name='the']/arr[@name='suggestions']/lst[2]/long[@name='weight'][.='2']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal']/lst[@name='the']/arr[@name='suggestions']/lst[2]/str[@name='payload'][.='lake']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal']/lst[@name='the']/arr[@name='suggestions']/lst[3]/str[@name='term'][.='star wars: episode v - <b>the</b> empire strikes back']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal']/lst[@name='the']/arr[@name='suggestions']/lst[3]/long[@name='weight'][.='0']",
+ "//lst[@name='suggest']/lst[@name='blended_infix_suggest_exponential_reciprocal']/lst[@name='the']/arr[@name='suggestions']/lst[3]/str[@name='payload'][.='star']"
+ );
+ }
+
public void testMultiSuggester() {
assertQ(req("qt", URI, "q", "the", SuggesterParams.SUGGEST_COUNT, "10", SuggesterParams.SUGGEST_DICT, "blended_infix_suggest_linear", SuggesterParams.SUGGEST_DICT, "blended_infix_suggest_reciprocal"),
"//lst[@name='suggest']/lst[@name='blended_infix_suggest_linear']/lst[@name='the']/int[@name='numFound'][.='3']",