You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mv...@apache.org on 2017/06/08 10:06:01 UTC
lucene-solr:branch_6x: LUCENE-7869: Changed MemoryIndex to sort 1d
points.
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 60f9c7b91 -> 9586b9195
LUCENE-7869: Changed MemoryIndex to sort 1d points.
In case of 1d points, the PointInSetQuery.MergePointVisitor expects that these points are visited in ascending order.
Prior to this change the memory index doesn't do this and this can result in document with multiple points that should match to not match.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9586b919
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9586b919
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9586b919
Branch: refs/heads/branch_6x
Commit: 9586b919563281d8e4c23452c90f096be5af0e63
Parents: 60f9c7b
Author: Martijn van Groningen <ma...@gmail.com>
Authored: Wed Jun 7 19:55:32 2017 +0200
Committer: Martijn van Groningen <ma...@gmail.com>
Committed: Thu Jun 8 12:05:47 2017 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +++
.../apache/lucene/index/memory/MemoryIndex.java | 35 ++++++++++++--------
.../lucene/index/memory/TestMemoryIndex.java | 24 ++++++++++++++
3 files changed, 49 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9586b919/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 7eafb45..f62982c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -15,6 +15,10 @@ Bug Fixes
* LUCENE-7864: IndexMergeTool is not using intermediate hard links (even
if possible). (Dawid Weiss)
+* LUCENE-7869: Changed MemoryIndex to sort 1d points. In case of 1d points, the PointInSetQuery.MergePointVisitor expects
+ that these points are visited in ascending order. The memory index doesn't do this and this can result in document
+ with multiple points that should match to not match. (Martijn van Groningen)
+
Other
* LUCENE-7800: Remove code that potentially rethrows checked exceptions
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9586b919/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index a1390b3..c8a4523 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -929,20 +929,27 @@ public class MemoryIndex {
final int numDimensions = fieldInfo.getPointDimensionCount();
final int numBytesPerDimension = fieldInfo.getPointNumBytes();
- minPackedValue = pointValues[0].bytes.clone();
- maxPackedValue = pointValues[0].bytes.clone();
-
- for (int i = 0; i < pointValuesCount; i++) {
- BytesRef pointValue = pointValues[i];
- assert pointValue.bytes.length == pointValue.length : "BytesRef should wrap a precise byte[], BytesRef.deepCopyOf() should take care of this";
-
- for (int dim = 0; dim < numDimensions; ++dim) {
- int offset = dim * numBytesPerDimension;
- if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, minPackedValue, offset) < 0) {
- System.arraycopy(pointValue.bytes, offset, minPackedValue, offset, numBytesPerDimension);
- }
- if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, maxPackedValue, offset) > 0) {
- System.arraycopy(pointValue.bytes, offset, maxPackedValue, offset, numBytesPerDimension);
+ if (numDimensions == 1) {
+ // PointInSetQuery.MergePointVisitor expects values to be visited in increasing order,
+ // this is a 1d optimization which has to be done here too. Otherwise we emit values
+ // out of order which causes mismatches.
+ Arrays.sort(pointValues, 0, pointValuesCount);
+ minPackedValue = pointValues[0].bytes.clone();
+ maxPackedValue = pointValues[pointValuesCount - 1].bytes.clone();
+ } else {
+ minPackedValue = pointValues[0].bytes.clone();
+ maxPackedValue = pointValues[0].bytes.clone();
+ for (int i = 0; i < pointValuesCount; i++) {
+ BytesRef pointValue = pointValues[i];
+ assert pointValue.bytes.length == pointValue.length : "BytesRef should wrap a precise byte[], BytesRef.deepCopyOf() should take care of this";
+ for (int dim = 0; dim < numDimensions; ++dim) {
+ int offset = dim * numBytesPerDimension;
+ if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, minPackedValue, offset) < 0) {
+ System.arraycopy(pointValue.bytes, offset, minPackedValue, offset, numBytesPerDimension);
+ }
+ if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, maxPackedValue, offset) > 0) {
+ System.arraycopy(pointValue.bytes, offset, maxPackedValue, offset, numBytesPerDimension);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9586b919/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
index 75344e3..8df1afb 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
@@ -476,6 +476,30 @@ public class TestMemoryIndex extends LuceneTestCase {
assertEquals(1, s.count(DoublePoint.newRangeQuery("doubles", new double[] {10D, 10D}, new double[] {30D, 30D})));
}
+ public void testMultiValuedPointsSortedCorrectly() throws Exception {
+ Document doc = new Document();
+ doc.add(new IntPoint("ints", 3));
+ doc.add(new IntPoint("ints", 2));
+ doc.add(new IntPoint("ints", 1));
+ doc.add(new LongPoint("longs", 3L));
+ doc.add(new LongPoint("longs", 2L));
+ doc.add(new LongPoint("longs", 1L));
+ doc.add(new FloatPoint("floats", 3F));
+ doc.add(new FloatPoint("floats", 2F));
+ doc.add(new FloatPoint("floats", 1F));
+ doc.add(new DoublePoint("doubles", 3D));
+ doc.add(new DoublePoint("doubles", 2D));
+ doc.add(new DoublePoint("doubles", 1D));
+
+ MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
+ IndexSearcher s = mi.createSearcher();
+
+ assertEquals(1, s.count(IntPoint.newSetQuery("ints", 2)));
+ assertEquals(1, s.count(LongPoint.newSetQuery("longs", 2)));
+ assertEquals(1, s.count(FloatPoint.newSetQuery("floats", 2)));
+ assertEquals(1, s.count(DoublePoint.newSetQuery("doubles", 2)));
+ }
+
public void testIndexingPointsAndDocValues() throws Exception {
FieldType type = new FieldType();
type.setDimensions(1, 4);