You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/07/15 23:45:34 UTC
svn commit: r1691283 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/suggest/
lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/
lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/
Author: mikemccand
Date: Wed Jul 15 21:45:33 2015
New Revision: 1691283
URL: http://svn.apache.org/r1691283
Log:
LUCENE-6680: don't lose a suggestion that differs only in payload from another suggestion
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java
lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1691283&r1=1691282&r2=1691283&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Wed Jul 15 21:45:33 2015
@@ -217,6 +217,9 @@ Bug fixes
* LUCENE-6677: QueryParserBase fails to enforce maxDeterminizedStates when
creating a WildcardQuery (David Causse via Mike McCandless)
+* LUCENE-6680: Preserve two suggestions that have same key and weight but
+ different payloads (Arcadius Ahouansou via Mike McCandless)
+
Changes in Runtime Behavior
* LUCENE-6501: The subreader structure in ParallelCompositeReader
Modified: lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java?rev=1691283&r1=1691282&r2=1691283&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java (original)
+++ lucene/dev/branches/branch_5x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java Wed Jul 15 21:45:33 2015
@@ -342,9 +342,19 @@ public class BlendedInfixSuggester exten
}
// otherwise on alphabetic order
- return CHARSEQUENCE_COMPARATOR.compare(o1.key, o2.key);
+ int keyCompare = CHARSEQUENCE_COMPARATOR.compare(o1.key, o2.key);
+
+ if (keyCompare != 0) {
+ return keyCompare;
+ }
+
+ // if same weight and title, use the payload if there is one
+ if (o1.payload != null) {
+ return o1.payload.compareTo(o2.payload);
+ }
+
+ return 0;
}
}
-
}
Modified: lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java?rev=1691283&r1=1691282&r2=1691283&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java (original)
+++ lucene/dev/branches/branch_5x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java Wed Jul 15 21:45:33 2015
@@ -22,6 +22,8 @@ import java.nio.file.Path;
import java.util.List;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.search.suggest.Input;
@@ -29,9 +31,11 @@ import org.apache.lucene.search.suggest.
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
public class BlendedInfixSuggesterTest extends LuceneTestCase {
+
/**
* Test the weight transformation depending on the position
* of the matching term.
@@ -192,6 +196,65 @@ public class BlendedInfixSuggesterTest e
suggester.close();
}
+ public void testBlendedInfixSuggesterDedupsOnWeightTitleAndPayload() throws Exception {
+
+ //exactly same inputs
+ Input[] inputDocuments = new Input[] {
+ new Input("lend me your ear", 7, new BytesRef("uid1")),
+ new Input("lend me your ear", 7, new BytesRef("uid1")),
+ };
+ duplicateCheck(inputDocuments, 1);
+
+ // inputs differ on payload
+ inputDocuments = new Input[] {
+ new Input("lend me your ear", 7, new BytesRef("uid1")),
+ new Input("lend me your ear", 7, new BytesRef("uid2")),
+ };
+ duplicateCheck(inputDocuments, 2);
+
+ //exactly same input without payloads
+ inputDocuments = new Input[] {
+ new Input("lend me your ear", 7),
+ new Input("lend me your ear", 7),
+ };
+ duplicateCheck(inputDocuments, 1);
+
+ //Same input with first has payloads, second does not
+ inputDocuments = new Input[] {
+ new Input("lend me your ear", 7, new BytesRef("uid1")),
+ new Input("lend me your ear", 7),
+ };
+ duplicateCheck(inputDocuments, 2);
+
+ /**same input, first not having a payload, the second having payload
+ * we would expect 2 entries out but we are getting only 1 because
+ * the InputArrayIterator#hasPayloads() returns false because the first
+ * item has no payload, therefore, when ingested, none of the 2 input has payload and become 1
+ */
+ inputDocuments = new Input[] {
+ new Input("lend me your ear", 7),
+ new Input("lend me your ear", 7, new BytesRef("uid2")),
+ };
+ List<Lookup.LookupResult> results = duplicateCheck(inputDocuments, 1);
+ assertNull(results.get(0).payload);
+
+
+ //exactly same inputs but different weight
+ inputDocuments = new Input[] {
+ new Input("lend me your ear", 1, new BytesRef("uid1")),
+ new Input("lend me your ear", 7, new BytesRef("uid1")),
+ };
+ duplicateCheck(inputDocuments, 2);
+
+ //exactly same inputs but different text
+ inputDocuments = new Input[] {
+ new Input("lend me your earings", 7, new BytesRef("uid1")),
+ new Input("lend me your ear", 7, new BytesRef("uid1")),
+ };
+ duplicateCheck(inputDocuments, 2);
+
+ }
+
public void /*testT*/rying() throws IOException {
BytesRef lake = new BytesRef("lake");
@@ -236,4 +299,23 @@ public class BlendedInfixSuggesterTest e
return -1;
}
+
+ private List<Lookup.LookupResult> duplicateCheck(Input[] inputs, int expectedSuggestionCount) throws IOException {
+
+ Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
+ BlendedInfixSuggester suggester = new BlendedInfixSuggester(newDirectory(), a, a, AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS,
+ BlendedInfixSuggester.BlenderType.POSITION_RECIPROCAL,10, false);
+
+ InputArrayIterator inputArrayIterator = new InputArrayIterator(inputs);
+ suggester.build(inputArrayIterator);
+
+ List<Lookup.LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), 10, true, true);
+ assertEquals(expectedSuggestionCount, results.size());
+
+ suggester.close();
+ a.close();
+
+ return results;
+ }
+
}