You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by an...@apache.org on 2019/09/25 22:26:08 UTC
[lucene-solr] branch branch_8x updated: LUCENE-8984: MoreLikeThis
MLT is biased for uncommon fields (#871) (#901)
This is an automated email from the ASF dual-hosted git repository.
anshum pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new 3c3d5b1 LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (#871) (#901)
3c3d5b1 is described below
commit 3c3d5b1172fe9221a44482a4a0ca04b9fd5f2246
Author: Anshum Gupta <an...@apache.org>
AuthorDate: Wed Sep 25 15:26:00 2019 -0700
LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (#871) (#901)
* LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (#871)
---
lucene/CHANGES.txt | 2 +
.../apache/lucene/queries/mlt/MoreLikeThis.java | 6 ++-
.../lucene/queries/mlt/TestMoreLikeThis.java | 59 ++++++++++++++++++++++
solr/CHANGES.txt | 2 +
4 files changed, 68 insertions(+), 1 deletion(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index be5e7ea..bdb819d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -81,6 +81,8 @@ Improvements
* LUCENE-8966: The Korean analyzer now splits tokens on boundaries between digits and alphabetic characters. (Jim Ferenczi)
+* LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (Andy Hind via Anshum Gupta)
+
Optimizations
* LUCENE-8922: DisjunctionMaxQuery more efficiently leverages impacts to skip
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java
index 7c077e5..37643b6 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java
@@ -649,13 +649,17 @@ public final class MoreLikeThis {
*/
private PriorityQueue<ScoreTerm> createQueue(Map<String, Map<String, Int>> perFieldTermFrequencies) throws IOException {
// have collected all words in doc and their freqs
- int numDocs = ir.numDocs();
final int limit = Math.min(maxQueryTerms, this.getTermsCount(perFieldTermFrequencies));
FreqQ queue = new FreqQ(limit); // will order words by score
for (Map.Entry<String, Map<String, Int>> entry : perFieldTermFrequencies.entrySet()) {
Map<String, Int> perWordTermFrequencies = entry.getValue();
String fieldName = entry.getKey();
+ long numDocs = ir.getDocCount(fieldName);
+ if(numDocs == -1) {
+ numDocs = ir.numDocs();
+ }
+
for (Map.Entry<String, Int> tfEntry : perWordTermFrequencies.entrySet()) { // for every word
String word = tfEntry.getKey();
int tf = tfEntry.getValue().x; // term freq in the source doc
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java b/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java
index 2061068..1c246f5 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -130,6 +131,64 @@ public class TestMoreLikeThis extends LuceneTestCase {
writer.addDocument(doc);
}
+ public void testSmallSampleFromCorpus() throws Throwable {
+ // add series of docs with terms of decreasing df
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+ for (int i = 0; i < 1980; i++) {
+ Document doc = new Document();
+ doc.add(newTextField("text", "filler", Field.Store.YES));
+ writer.addDocument(doc);
+ }
+ for (int i = 0; i < 18; i++) {
+ Document doc = new Document();
+ doc.add(newTextField("one_percent", "all", Field.Store.YES));
+ writer.addDocument(doc);
+ }
+ for (int i = 0; i < 2; i++) {
+ Document doc = new Document();
+ doc.add(newTextField("one_percent", "all", Field.Store.YES));
+ doc.add(newTextField("one_percent", "tenth", Field.Store.YES));
+ writer.addDocument(doc);
+ }
+ IndexReader reader = writer.getReader();
+ writer.close();
+
+ // setup MLT query
+ MoreLikeThis mlt = new MoreLikeThis(reader);
+ Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
+ mlt.setAnalyzer(analyzer);
+ mlt.setMaxQueryTerms(3);
+ mlt.setMinDocFreq(1);
+ mlt.setMinTermFreq(1);
+ mlt.setMinWordLen(1);
+ mlt.setFieldNames(new String[]{"one_percent"});
+
+ BooleanQuery query = (BooleanQuery) mlt.like("one_percent", new StringReader("tenth tenth all"));
+ Collection<BooleanClause> clauses = query.clauses();
+
+ assertTrue(clauses.size() == 2);
+ Term term = ((TermQuery) ((List<BooleanClause>) clauses).get(0).getQuery()).getTerm();
+ assertTrue(term.text().equals("all"));
+ term = ((TermQuery) ((List<BooleanClause>) clauses).get(1).getQuery()).getTerm();
+ assertTrue(term.text().equals("tenth"));
+
+
+ query = (BooleanQuery) mlt.like("one_percent", new StringReader("tenth all all"));
+ clauses = query.clauses();
+
+ assertTrue(clauses.size() == 2);
+ term = ((TermQuery) ((List<BooleanClause>) clauses).get(0).getQuery()).getTerm();
+ assertTrue(term.text().equals("all"));
+ term = ((TermQuery) ((List<BooleanClause>) clauses).get(1).getQuery()).getTerm();
+ assertTrue(term.text().equals("tenth"));
+
+ // clean up
+ reader.close();
+ dir.close();
+ analyzer.close();
+ }
+
public void testBoostFactor() throws Throwable {
Map<String,Float> originalValues = getOriginalValues();
mlt.setFieldNames(new String[] {"text"});
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 48f250e..c012657 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -121,6 +121,8 @@ Improvements
* SOLR-13784: EmbeddedSolrServer's defaultCore constructor argument is now optional (David Smiley)
+* LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (Andy Hind via Anshum Gupta)
+
Bug Fixes
----------------------