You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2016/06/16 13:58:43 UTC
[1/2] lucene-solr:branch_5_5: LUCENE-7065: Fix the explain for the
global ordinals join query. Before the explain would also indicate that non
matching documents would match. On top of that with score mode average,
the explain would fail with a NPE.
Repository: lucene-solr
Updated Branches:
refs/heads/branch_5_5 489f386b7 -> dac7ea817
refs/heads/branch_5x 0442aa5e5 -> a92f330a4
LUCENE-7065: Fix the explain for the global ordinals join query. Before the
explain would also indicate that non matching documents would match.
On top of that with score mode average, the explain would fail with a NPE.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/dac7ea81
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/dac7ea81
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/dac7ea81
Branch: refs/heads/branch_5_5
Commit: dac7ea81704ef708e37b9e7902a29a8e2243b183
Parents: 489f386
Author: Martijn van Groningen <mv...@apache.org>
Authored: Fri Mar 4 18:23:00 2016 +0100
Committer: Steve Rowe <sa...@apache.org>
Committed: Thu Jun 16 09:56:55 2016 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 5 +
.../lucene/search/join/GlobalOrdinalsQuery.java | 27 +++--
.../join/GlobalOrdinalsWithScoreQuery.java | 35 ++++---
.../apache/lucene/search/join/TestJoinUtil.java | 103 +++++++++++++++++++
4 files changed, 149 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/dac7ea81/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 3c5ef91..64cb1d5 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -7,6 +7,11 @@ http://s.apache.org/luceneversions
Bug Fixes
+* LUCENE-7065: Fix the explain for the global ordinals join query. Before the
+ explain would also indicate that non matching documents would match.
+ On top of that with score mode average, the explain would fail with a NPE.
+ (Martijn van Groningen)
+
* LUCENE-7187: Block join queries' Weight#extractTerms(...) implementations
should delegate to the wrapped weight. (Martijn van Groningen)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/dac7ea81/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
index e1ea978..135866c 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
@@ -113,14 +113,27 @@ final class GlobalOrdinalsQuery extends Query {
@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
- if (values != null) {
- int segmentOrd = values.getOrd(doc);
- if (segmentOrd != -1) {
- BytesRef joinValue = values.lookupOrd(segmentOrd);
- return Explanation.match(score(), "Score based on join value " + joinValue.utf8ToString());
- }
+ if (values == null) {
+ return Explanation.noMatch("Not a match");
}
- return Explanation.noMatch("Not a match");
+
+ int segmentOrd = values.getOrd(doc);
+ if (segmentOrd == -1) {
+ return Explanation.noMatch("Not a match");
+ }
+ BytesRef joinValue = values.lookupOrd(segmentOrd);
+
+ int ord;
+ if (globalOrds != null) {
+ ord = (int) globalOrds.getGlobalOrds(context.ord).get(segmentOrd);
+ } else {
+ ord = segmentOrd;
+ }
+ if (foundOrds.get(ord) == false) {
+ return Explanation.noMatch("Not a match, join value " + Term.toString(joinValue));
+ }
+
+ return Explanation.match(score(), "A match, join value " + Term.toString(joinValue));
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/dac7ea81/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
index 6da3609..110b8a3 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
@@ -121,21 +121,28 @@ final class GlobalOrdinalsWithScoreQuery extends Query {
@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
- if (values != null) {
- int segmentOrd = values.getOrd(doc);
- if (segmentOrd != -1) {
- final float score;
- if (globalOrds != null) {
- long globalOrd = globalOrds.getGlobalOrds(context.ord).get(segmentOrd);
- score = collector.score((int) globalOrd);
- } else {
- score = collector.score(segmentOrd);
- }
- BytesRef joinValue = values.lookupOrd(segmentOrd);
- return Explanation.match(score, "Score based on join value " + joinValue.utf8ToString());
- }
+ if (values == null) {
+ return Explanation.noMatch("Not a match");
+ }
+
+ int segmentOrd = values.getOrd(doc);
+ if (segmentOrd == -1) {
+ return Explanation.noMatch("Not a match");
}
- return Explanation.noMatch("Not a match");
+ BytesRef joinValue = values.lookupOrd(segmentOrd);
+
+ int ord;
+ if (globalOrds != null) {
+ ord = (int) globalOrds.getGlobalOrds(context.ord).get(segmentOrd);
+ } else {
+ ord = segmentOrd;
+ }
+ if (collector.match(ord) == false) {
+ return Explanation.noMatch("Not a match, join value " + Term.toString(joinValue));
+ }
+
+ float score = collector.score(ord);
+ return Explanation.match(score, "A match, join value " + Term.toString(joinValue));
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/dac7ea81/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
index 98b8395..a069d03 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
@@ -51,6 +51,7 @@ import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
@@ -297,6 +298,108 @@ public class TestJoinUtil extends LuceneTestCase {
dir.close();
}
+ public void testOrdinalsJoinExplainNoMatches() throws Exception {
+ final String idField = "id";
+ final String productIdField = "productId";
+ // A field indicating to what type a document belongs, which is then used to distinques between documents during joining.
+ final String typeField = "type";
+ // A single sorted doc values field that holds the join values for all document types.
+ // Typically during indexing a schema will automatically create this field with the values
+ final String joinField = idField + productIdField;
+
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(
+ dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE)
+ );
+
+ // 0
+ Document doc = new Document();
+ doc.add(new TextField(idField, "1", Field.Store.NO));
+ doc.add(new TextField(typeField, "product", Field.Store.NO));
+ doc.add(new TextField("description", "random text", Field.Store.NO));
+ doc.add(new TextField("name", "name1", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("1")));
+ w.addDocument(doc);
+
+ // 1
+ doc = new Document();
+ doc.add(new TextField(idField, "2", Field.Store.NO));
+ doc.add(new TextField(typeField, "product", Field.Store.NO));
+ doc.add(new TextField("description", "random text", Field.Store.NO));
+ doc.add(new TextField("name", "name2", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("2")));
+ w.addDocument(doc);
+
+ // 2
+ doc = new Document();
+ doc.add(new TextField(productIdField, "1", Field.Store.NO));
+ doc.add(new TextField(typeField, "price", Field.Store.NO));
+ doc.add(new TextField("price", "10.0", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("1")));
+ w.addDocument(doc);
+
+ // 3
+ doc = new Document();
+ doc.add(new TextField(productIdField, "2", Field.Store.NO));
+ doc.add(new TextField(typeField, "price", Field.Store.NO));
+ doc.add(new TextField("price", "20.0", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("1")));
+ w.addDocument(doc);
+
+ if (random().nextBoolean()) {
+ w.flush();
+ }
+
+ // 4
+ doc = new Document();
+ doc.add(new TextField(productIdField, "3", Field.Store.NO));
+ doc.add(new TextField(typeField, "price", Field.Store.NO));
+ doc.add(new TextField("price", "5.0", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("2")));
+ w.addDocument(doc);
+
+ // 5
+ doc = new Document();
+ doc.add(new TextField("field", "value", Field.Store.NO));
+ w.addDocument(doc);
+
+ IndexReader r = DirectoryReader.open(w);
+ IndexSearcher indexSearcher = new IndexSearcher(r);
+ SortedDocValues[] values = new SortedDocValues[r.leaves().size()];
+ for (int i = 0; i < values.length; i++) {
+ LeafReader leafReader = r.leaves().get(i).reader();
+ values[i] = DocValues.getSorted(leafReader, joinField);
+ }
+ MultiDocValues.OrdinalMap ordinalMap = MultiDocValues.OrdinalMap.build(
+ r.getCoreCacheKey(), values, PackedInts.DEFAULT
+ );
+
+ Query toQuery = new TermQuery(new Term("price", "5.0"));
+ Query fromQuery = new TermQuery(new Term("name", "name2"));
+
+ for (ScoreMode scoreMode : ScoreMode.values()) {
+ Query joinQuery = JoinUtil.createJoinQuery(joinField, fromQuery, toQuery, indexSearcher, scoreMode, ordinalMap);
+ TopDocs result = indexSearcher.search(joinQuery, 10);
+ assertEquals(1, result.totalHits);
+ assertEquals(4, result.scoreDocs[0].doc); // doc with price: 5.0
+ Explanation explanation = indexSearcher.explain(joinQuery, 4);
+ assertTrue(explanation.isMatch());
+ assertEquals(explanation.getDescription(), "A match, join value 2");
+
+ explanation = indexSearcher.explain(joinQuery, 3);
+ assertFalse(explanation.isMatch());
+ assertEquals(explanation.getDescription(), "Not a match, join value 1");
+
+ explanation = indexSearcher.explain(joinQuery, 5);
+ assertFalse(explanation.isMatch());
+ assertEquals(explanation.getDescription(), "Not a match");
+ }
+
+ w.close();
+ indexSearcher.getIndexReader().close();
+ dir.close();
+ }
+
public void testRandomOrdinalsJoin() throws Exception {
IndexIterationContext context = createContext(512, false, true);
int searchIters = 10;
[2/2] lucene-solr:branch_5x: LUCENE-7065: Fix the explain for the
global ordinals join query. Before the explain would also indicate that non
matching documents would match. On top of that with score mode average,
the explain would fail with a NPE.
Posted by sa...@apache.org.
LUCENE-7065: Fix the explain for the global ordinals join query. Before the
explain would also indicate that non matching documents would match.
On top of that with score mode average, the explain would fail with a NPE.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a92f330a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a92f330a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a92f330a
Branch: refs/heads/branch_5x
Commit: a92f330a42ee898cc5ceaa5b0676f5fb7d2b5d9e
Parents: 0442aa5
Author: Martijn van Groningen <mv...@apache.org>
Authored: Fri Mar 4 18:23:00 2016 +0100
Committer: Steve Rowe <sa...@apache.org>
Committed: Thu Jun 16 09:57:45 2016 -0400
----------------------------------------------------------------------
.../lucene/search/join/GlobalOrdinalsQuery.java | 27 +++--
.../join/GlobalOrdinalsWithScoreQuery.java | 35 ++++---
.../apache/lucene/search/join/TestJoinUtil.java | 103 +++++++++++++++++++
3 files changed, 144 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a92f330a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
index e1ea978..135866c 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
@@ -113,14 +113,27 @@ final class GlobalOrdinalsQuery extends Query {
@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
- if (values != null) {
- int segmentOrd = values.getOrd(doc);
- if (segmentOrd != -1) {
- BytesRef joinValue = values.lookupOrd(segmentOrd);
- return Explanation.match(score(), "Score based on join value " + joinValue.utf8ToString());
- }
+ if (values == null) {
+ return Explanation.noMatch("Not a match");
}
- return Explanation.noMatch("Not a match");
+
+ int segmentOrd = values.getOrd(doc);
+ if (segmentOrd == -1) {
+ return Explanation.noMatch("Not a match");
+ }
+ BytesRef joinValue = values.lookupOrd(segmentOrd);
+
+ int ord;
+ if (globalOrds != null) {
+ ord = (int) globalOrds.getGlobalOrds(context.ord).get(segmentOrd);
+ } else {
+ ord = segmentOrd;
+ }
+ if (foundOrds.get(ord) == false) {
+ return Explanation.noMatch("Not a match, join value " + Term.toString(joinValue));
+ }
+
+ return Explanation.match(score(), "A match, join value " + Term.toString(joinValue));
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a92f330a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
index 6da3609..110b8a3 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
@@ -121,21 +121,28 @@ final class GlobalOrdinalsWithScoreQuery extends Query {
@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
- if (values != null) {
- int segmentOrd = values.getOrd(doc);
- if (segmentOrd != -1) {
- final float score;
- if (globalOrds != null) {
- long globalOrd = globalOrds.getGlobalOrds(context.ord).get(segmentOrd);
- score = collector.score((int) globalOrd);
- } else {
- score = collector.score(segmentOrd);
- }
- BytesRef joinValue = values.lookupOrd(segmentOrd);
- return Explanation.match(score, "Score based on join value " + joinValue.utf8ToString());
- }
+ if (values == null) {
+ return Explanation.noMatch("Not a match");
+ }
+
+ int segmentOrd = values.getOrd(doc);
+ if (segmentOrd == -1) {
+ return Explanation.noMatch("Not a match");
}
- return Explanation.noMatch("Not a match");
+ BytesRef joinValue = values.lookupOrd(segmentOrd);
+
+ int ord;
+ if (globalOrds != null) {
+ ord = (int) globalOrds.getGlobalOrds(context.ord).get(segmentOrd);
+ } else {
+ ord = segmentOrd;
+ }
+ if (collector.match(ord) == false) {
+ return Explanation.noMatch("Not a match, join value " + Term.toString(joinValue));
+ }
+
+ float score = collector.score(ord);
+ return Explanation.match(score, "A match, join value " + Term.toString(joinValue));
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a92f330a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
index 98b8395..a069d03 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
@@ -51,6 +51,7 @@ import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
@@ -297,6 +298,108 @@ public class TestJoinUtil extends LuceneTestCase {
dir.close();
}
+ public void testOrdinalsJoinExplainNoMatches() throws Exception {
+ final String idField = "id";
+ final String productIdField = "productId";
+ // A field indicating to what type a document belongs, which is then used to distinques between documents during joining.
+ final String typeField = "type";
+ // A single sorted doc values field that holds the join values for all document types.
+ // Typically during indexing a schema will automatically create this field with the values
+ final String joinField = idField + productIdField;
+
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(
+ dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE)
+ );
+
+ // 0
+ Document doc = new Document();
+ doc.add(new TextField(idField, "1", Field.Store.NO));
+ doc.add(new TextField(typeField, "product", Field.Store.NO));
+ doc.add(new TextField("description", "random text", Field.Store.NO));
+ doc.add(new TextField("name", "name1", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("1")));
+ w.addDocument(doc);
+
+ // 1
+ doc = new Document();
+ doc.add(new TextField(idField, "2", Field.Store.NO));
+ doc.add(new TextField(typeField, "product", Field.Store.NO));
+ doc.add(new TextField("description", "random text", Field.Store.NO));
+ doc.add(new TextField("name", "name2", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("2")));
+ w.addDocument(doc);
+
+ // 2
+ doc = new Document();
+ doc.add(new TextField(productIdField, "1", Field.Store.NO));
+ doc.add(new TextField(typeField, "price", Field.Store.NO));
+ doc.add(new TextField("price", "10.0", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("1")));
+ w.addDocument(doc);
+
+ // 3
+ doc = new Document();
+ doc.add(new TextField(productIdField, "2", Field.Store.NO));
+ doc.add(new TextField(typeField, "price", Field.Store.NO));
+ doc.add(new TextField("price", "20.0", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("1")));
+ w.addDocument(doc);
+
+ if (random().nextBoolean()) {
+ w.flush();
+ }
+
+ // 4
+ doc = new Document();
+ doc.add(new TextField(productIdField, "3", Field.Store.NO));
+ doc.add(new TextField(typeField, "price", Field.Store.NO));
+ doc.add(new TextField("price", "5.0", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("2")));
+ w.addDocument(doc);
+
+ // 5
+ doc = new Document();
+ doc.add(new TextField("field", "value", Field.Store.NO));
+ w.addDocument(doc);
+
+ IndexReader r = DirectoryReader.open(w);
+ IndexSearcher indexSearcher = new IndexSearcher(r);
+ SortedDocValues[] values = new SortedDocValues[r.leaves().size()];
+ for (int i = 0; i < values.length; i++) {
+ LeafReader leafReader = r.leaves().get(i).reader();
+ values[i] = DocValues.getSorted(leafReader, joinField);
+ }
+ MultiDocValues.OrdinalMap ordinalMap = MultiDocValues.OrdinalMap.build(
+ r.getCoreCacheKey(), values, PackedInts.DEFAULT
+ );
+
+ Query toQuery = new TermQuery(new Term("price", "5.0"));
+ Query fromQuery = new TermQuery(new Term("name", "name2"));
+
+ for (ScoreMode scoreMode : ScoreMode.values()) {
+ Query joinQuery = JoinUtil.createJoinQuery(joinField, fromQuery, toQuery, indexSearcher, scoreMode, ordinalMap);
+ TopDocs result = indexSearcher.search(joinQuery, 10);
+ assertEquals(1, result.totalHits);
+ assertEquals(4, result.scoreDocs[0].doc); // doc with price: 5.0
+ Explanation explanation = indexSearcher.explain(joinQuery, 4);
+ assertTrue(explanation.isMatch());
+ assertEquals(explanation.getDescription(), "A match, join value 2");
+
+ explanation = indexSearcher.explain(joinQuery, 3);
+ assertFalse(explanation.isMatch());
+ assertEquals(explanation.getDescription(), "Not a match, join value 1");
+
+ explanation = indexSearcher.explain(joinQuery, 5);
+ assertFalse(explanation.isMatch());
+ assertEquals(explanation.getDescription(), "Not a match");
+ }
+
+ w.close();
+ indexSearcher.getIndexReader().close();
+ dir.close();
+ }
+
public void testRandomOrdinalsJoin() throws Exception {
IndexIterationContext context = createContext(512, false, true);
int searchIters = 10;