You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mv...@apache.org on 2017/06/08 10:05:16 UTC
lucene-solr:master: LUCENE-7869: Changed MemoryIndex to sort 1d
points.
Repository: lucene-solr
Updated Branches:
refs/heads/master cf9b70b25 -> 5a34d419b
LUCENE-7869: Changed MemoryIndex to sort 1d points.
In case of 1d points, the PointInSetQuery.MergePointVisitor expects that these points are visited in ascending order.
Prior to this change the memory index doesn't do this and this can result in document with multiple points that should match to not match.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5a34d419
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5a34d419
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5a34d419
Branch: refs/heads/master
Commit: 5a34d419b9afea443e56a775f3b7f2d08e18add7
Parents: cf9b70b
Author: Martijn van Groningen <ma...@gmail.com>
Authored: Wed Jun 7 19:55:32 2017 +0200
Committer: Martijn van Groningen <ma...@gmail.com>
Committed: Thu Jun 8 11:52:24 2017 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +++
.../apache/lucene/index/memory/MemoryIndex.java | 35 ++++++++++++--------
.../lucene/index/memory/TestMemoryIndex.java | 24 ++++++++++++++
3 files changed, 49 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5a34d419/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0bc032b..0251243 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -129,6 +129,10 @@ Bug Fixes
* LUCENE-7864: IndexMergeTool is not using intermediate hard links (even
if possible). (Dawid Weiss)
+* LUCENE-7869: Changed MemoryIndex to sort 1d points. In case of 1d points, the PointInSetQuery.MergePointVisitor expects
+ that these points are visited in ascending order. The memory index doesn't do this and this can result in document
+ with multiple points that should match to not match. (Martijn van Groningen)
+
Other
* LUCENE-7800: Remove code that potentially rethrows checked exceptions
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5a34d419/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index a409a17..92b9359 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -868,20 +868,27 @@ public class MemoryIndex {
final int numDimensions = fieldInfo.getPointDimensionCount();
final int numBytesPerDimension = fieldInfo.getPointNumBytes();
- minPackedValue = pointValues[0].bytes.clone();
- maxPackedValue = pointValues[0].bytes.clone();
-
- for (int i = 0; i < pointValuesCount; i++) {
- BytesRef pointValue = pointValues[i];
- assert pointValue.bytes.length == pointValue.length : "BytesRef should wrap a precise byte[], BytesRef.deepCopyOf() should take care of this";
-
- for (int dim = 0; dim < numDimensions; ++dim) {
- int offset = dim * numBytesPerDimension;
- if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, minPackedValue, offset) < 0) {
- System.arraycopy(pointValue.bytes, offset, minPackedValue, offset, numBytesPerDimension);
- }
- if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, maxPackedValue, offset) > 0) {
- System.arraycopy(pointValue.bytes, offset, maxPackedValue, offset, numBytesPerDimension);
+ if (numDimensions == 1) {
+ // PointInSetQuery.MergePointVisitor expects values to be visited in increasing order,
+ // this is a 1d optimization which has to be done here too. Otherwise we emit values
+ // out of order which causes mismatches.
+ Arrays.sort(pointValues, 0, pointValuesCount);
+ minPackedValue = pointValues[0].bytes.clone();
+ maxPackedValue = pointValues[pointValuesCount - 1].bytes.clone();
+ } else {
+ minPackedValue = pointValues[0].bytes.clone();
+ maxPackedValue = pointValues[0].bytes.clone();
+ for (int i = 0; i < pointValuesCount; i++) {
+ BytesRef pointValue = pointValues[i];
+ assert pointValue.bytes.length == pointValue.length : "BytesRef should wrap a precise byte[], BytesRef.deepCopyOf() should take care of this";
+ for (int dim = 0; dim < numDimensions; ++dim) {
+ int offset = dim * numBytesPerDimension;
+ if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, minPackedValue, offset) < 0) {
+ System.arraycopy(pointValue.bytes, offset, minPackedValue, offset, numBytesPerDimension);
+ }
+ if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, maxPackedValue, offset) > 0) {
+ System.arraycopy(pointValue.bytes, offset, maxPackedValue, offset, numBytesPerDimension);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5a34d419/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
index 3d3a12c..fa62262 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
@@ -512,6 +512,30 @@ public class TestMemoryIndex extends LuceneTestCase {
assertEquals(1, s.count(DoublePoint.newRangeQuery("doubles", new double[] {10D, 10D}, new double[] {30D, 30D})));
}
+ public void testMultiValuedPointsSortedCorrectly() throws Exception {
+ Document doc = new Document();
+ doc.add(new IntPoint("ints", 3));
+ doc.add(new IntPoint("ints", 2));
+ doc.add(new IntPoint("ints", 1));
+ doc.add(new LongPoint("longs", 3L));
+ doc.add(new LongPoint("longs", 2L));
+ doc.add(new LongPoint("longs", 1L));
+ doc.add(new FloatPoint("floats", 3F));
+ doc.add(new FloatPoint("floats", 2F));
+ doc.add(new FloatPoint("floats", 1F));
+ doc.add(new DoublePoint("doubles", 3D));
+ doc.add(new DoublePoint("doubles", 2D));
+ doc.add(new DoublePoint("doubles", 1D));
+
+ MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
+ IndexSearcher s = mi.createSearcher();
+
+ assertEquals(1, s.count(IntPoint.newSetQuery("ints", 2)));
+ assertEquals(1, s.count(LongPoint.newSetQuery("longs", 2)));
+ assertEquals(1, s.count(FloatPoint.newSetQuery("floats", 2)));
+ assertEquals(1, s.count(DoublePoint.newSetQuery("doubles", 2)));
+ }
+
public void testIndexingPointsAndDocValues() throws Exception {
FieldType type = new FieldType();
type.setDimensions(1, 4);