You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by th...@apache.org on 2016/04/14 18:51:13 UTC
[1/7] lucene-solr:jira/SOLR-8908: improve exception messages when
requested slice is out of bounds
Repository: lucene-solr
Updated Branches:
refs/heads/jira/SOLR-8908 a3b4e582a -> 4db9f6bfe
improve exception messages when requested slice is out of bounds
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/60033b30
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/60033b30
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/60033b30
Branch: refs/heads/jira/SOLR-8908
Commit: 60033b308ac721aff340b86d6776ce47cfbb63aa
Parents: 2335a45
Author: Mike McCandless <mi...@apache.org>
Authored: Thu Apr 14 05:52:53 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Thu Apr 14 05:52:53 2016 -0400
----------------------------------------------------------------------
lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java | 2 +-
.../core/src/java/org/apache/lucene/store/SimpleFSDirectory.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/60033b30/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
index 751e14e..27a90e8 100644
--- a/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
@@ -134,7 +134,7 @@ public class NIOFSDirectory extends FSDirectory {
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
if (offset < 0 || length < 0 || offset + length > this.length()) {
- throw new IllegalArgumentException("slice() " + sliceDescription + " out of bounds: " + this);
+ throw new IllegalArgumentException("slice() " + sliceDescription + " out of bounds: offset=" + offset + ",length=" + length + ",fileLength=" + this.length() + ": " + this);
}
return new NIOFSIndexInput(getFullSliceDescription(sliceDescription), channel, off + offset, length, getBufferSize());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/60033b30/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
index 87fde83..0d650ae 100644
--- a/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
@@ -130,7 +130,7 @@ public class SimpleFSDirectory extends FSDirectory {
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
if (offset < 0 || length < 0 || offset + length > this.length()) {
- throw new IllegalArgumentException("slice() " + sliceDescription + " out of bounds: " + this);
+ throw new IllegalArgumentException("slice() " + sliceDescription + " out of bounds: offset=" + offset + ",length=" + length + ",fileLength=" + this.length() + ": " + this);
}
return new SimpleFSIndexInput(getFullSliceDescription(sliceDescription), channel, off + offset, length, getBufferSize());
}
[2/7] lucene-solr:jira/SOLR-8908: LUCENE-7218: remove wrong comment
Posted by th...@apache.org.
LUCENE-7218: remove wrong comment
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/af5742cc
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/af5742cc
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/af5742cc
Branch: refs/heads/jira/SOLR-8908
Commit: af5742cc0abe09ec00c6550387d7bedab5a10056
Parents: 60033b3
Author: Mike McCandless <mi...@apache.org>
Authored: Thu Apr 14 05:55:15 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Thu Apr 14 05:55:15 2016 -0400
----------------------------------------------------------------------
.../java/org/apache/lucene/replicator/http/ReplicationService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af5742cc/lucene/replicator/src/java/org/apache/lucene/replicator/http/ReplicationService.java
----------------------------------------------------------------------
diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/http/ReplicationService.java b/lucene/replicator/src/java/org/apache/lucene/replicator/http/ReplicationService.java
index 4cb4368..e392445 100644
--- a/lucene/replicator/src/java/org/apache/lucene/replicator/http/ReplicationService.java
+++ b/lucene/replicator/src/java/org/apache/lucene/replicator/http/ReplicationService.java
@@ -171,7 +171,7 @@ public class ReplicationService {
if (token == null) {
resOut.write(0); // marker for null token
} else {
- resOut.write(1); // marker for null token
+ resOut.write(1);
token.serialize(new DataOutputStream(resOut));
}
break;
[7/7] lucene-solr:jira/SOLR-8908: Update CHANGES.txt to mention fix
for SOLR-8908
Posted by th...@apache.org.
Update CHANGES.txt to mention fix for SOLR-8908
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4db9f6bf
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4db9f6bf
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4db9f6bf
Branch: refs/heads/jira/SOLR-8908
Commit: 4db9f6bfeaad38f66877ebd6e6b5a58683644842
Parents: 18eb7f4
Author: Timothy Potter <th...@gmail.com>
Authored: Thu Apr 14 09:51:06 2016 -0700
Committer: Timothy Potter <th...@gmail.com>
Committed: Thu Apr 14 09:51:06 2016 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4db9f6bf/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index b2b7ecd..f40bc95 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -609,6 +609,9 @@ Bug Fixes
* SOLR-8870: AngularJS Query tab no longer URL-encodes the /select part of the request, fixing possible 404 issue
when Solr is behind a proxy. Also, now supports old-style &qt param when handler not prefixed with "/" (janhoy)
+* SOLR-8908: Fix to OnReconnect listener registration to allow listeners to deregister, such
+ as when a core is reloaded or deleted to avoid a memory leak. (Timothy Potter)
+
======================= 5.5.0 =======================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release
[6/7] lucene-solr:jira/SOLR-8908: Merge branch 'master' into
jira/SOLR-8908
Posted by th...@apache.org.
Merge branch 'master' into jira/SOLR-8908
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/18eb7f4c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/18eb7f4c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/18eb7f4c
Branch: refs/heads/jira/SOLR-8908
Commit: 18eb7f4c47a5f10ea878b8ca367c77a61b0f540c
Parents: a3b4e58 5238de9
Author: Timothy Potter <th...@gmail.com>
Authored: Thu Apr 14 09:35:29 2016 -0700
Committer: Timothy Potter <th...@gmail.com>
Committed: Thu Apr 14 09:35:29 2016 -0700
----------------------------------------------------------------------
lucene/CHANGES.txt | 6 +
.../codecs/lucene60/Lucene60PointsReader.java | 5 +-
.../org/apache/lucene/store/NIOFSDirectory.java | 2 +-
.../apache/lucene/store/SimpleFSDirectory.java | 2 +-
.../org/apache/lucene/util/bkd/BKDReader.java | 47 ++-
.../xml/CoreParserTestIndexData.java | 74 +++++
.../lucene/queryparser/xml/TestCoreParser.java | 89 +++---
.../xml/TestCorePlusExtensionsParser.java | 17 +-
.../xml/TestCorePlusQueriesParser.java | 17 +-
.../replicator/http/ReplicationService.java | 2 +-
.../org/apache/lucene/document/LatLonPoint.java | 57 +++-
.../apache/lucene/document/NearestNeighbor.java | 308 +++++++++++++++++++
.../org/apache/lucene/document/TestNearest.java | 245 +++++++++++++++
13 files changed, 776 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
[5/7] lucene-solr:jira/SOLR-8908: LUCENE-7069: woops,
approxBestDistance was way too approximate when the point was inside
the cell
Posted by th...@apache.org.
LUCENE-7069: woops, approxBestDistance was way too approximate when the point was inside the cell
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5238de93
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5238de93
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5238de93
Branch: refs/heads/jira/SOLR-8908
Commit: 5238de937a84c4de387f0036830811cb3b7d734f
Parents: 8d5d201
Author: Mike McCandless <mi...@apache.org>
Authored: Thu Apr 14 12:05:10 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Thu Apr 14 12:05:10 2016 -0400
----------------------------------------------------------------------
.../apache/lucene/document/NearestNeighbor.java | 18 +++++-------------
1 file changed, 5 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5238de93/lucene/sandbox/src/java/org/apache/lucene/document/NearestNeighbor.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/NearestNeighbor.java b/lucene/sandbox/src/java/org/apache/lucene/document/NearestNeighbor.java
index 581be46..3b9f302 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/NearestNeighbor.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/NearestNeighbor.java
@@ -66,7 +66,7 @@ class NearestNeighbor {
double minLon = decodeLongitude(minPacked, Integer.BYTES);
double maxLat = decodeLatitude(maxPacked, 0);
double maxLon = decodeLongitude(maxPacked, Integer.BYTES);
- return "Cell(readerIndex=" + readerIndex + " lat=" + minLat + " TO " + maxLat + ", lon=" + minLon + " TO " + maxLon + ")";
+ return "Cell(readerIndex=" + readerIndex + " lat=" + minLat + " TO " + maxLat + ", lon=" + minLon + " TO " + maxLon + "; distanceMeters=" + distanceMeters + ")";
}
}
@@ -88,8 +88,6 @@ class NearestNeighbor {
// second set of longitude ranges to check (for cross-dateline case)
private double minLon2 = Double.POSITIVE_INFINITY;
- int pointCheckCount;
-
public NearestVisitor(PriorityQueue<NearestHit> hitQueue, int topN, double pointLat, double pointLon) {
this.hitQueue = hitQueue;
this.topN = topN;
@@ -106,6 +104,7 @@ class NearestNeighbor {
if (setBottomCounter < 1024 || (setBottomCounter & 0x3F) == 0x3F) {
NearestHit hit = hitQueue.peek();
Rectangle box = Rectangle.fromPointDistance(pointLat, pointLon, hit.distanceMeters);
+ //System.out.println(" update bbox to " + box);
minLat = box.minLat;
maxLat = box.maxLat;
if (box.crossesDateline()) {
@@ -145,8 +144,6 @@ class NearestNeighbor {
return;
}
- pointCheckCount++;
-
double distanceMeters = SloppyMath.haversinMeters(pointLat, pointLon, docLatitude, docLongitude);
//System.out.println(" visit docID=" + docID + " distanceMeters=" + distanceMeters + " docLat=" + docLatitude + " docLon=" + docLongitude);
@@ -198,7 +195,7 @@ class NearestNeighbor {
public static NearestHit[] nearest(double pointLat, double pointLon, List<BKDReader> readers, List<Bits> liveDocs, List<Integer> docBases, final int n) throws IOException {
- //System.out.println("NEAREST: r=" + r + " liveDocs=" + liveDocs);
+ //System.out.println("NEAREST: readers=" + readers + " liveDocs=" + liveDocs + " pointLat=" + pointLat + " pointLon=" + pointLon);
// Holds closest collected points seen so far:
// TODO: if we used lucene's PQ we could just updateTop instead of poll/offer:
final PriorityQueue<NearestHit> hitQueue = new PriorityQueue<>(n, new Comparator<NearestHit>() {
@@ -225,16 +222,11 @@ class NearestNeighbor {
for(int i=0;i<readers.size();i++) {
BKDReader reader = readers.get(i);
byte[] minPackedValue = reader.getMinPackedValue();
- double minLat = decodeLatitude(minPackedValue, 0);
- double minLon = decodeLongitude(minPackedValue, Integer.BYTES);
-
byte[] maxPackedValue = reader.getMaxPackedValue();
- double maxLat = decodeLatitude(maxPackedValue, 0);
- double maxLon = decodeLongitude(maxPackedValue, Integer.BYTES);
states.add(reader.getIntersectState(visitor));
cellQueue.offer(new Cell(i, 1, reader.getMinPackedValue(), reader.getMaxPackedValue(),
- approxBestDistance(minLat, maxLat, minLon, maxLon, pointLat, pointLon)));
+ approxBestDistance(minPackedValue, maxPackedValue, pointLat, pointLon)));
}
while (cellQueue.size() > 0) {
@@ -302,7 +294,7 @@ class NearestNeighbor {
// TODO: can we make this the trueBestDistance? I.e., minimum distance between the point and ANY point on the box? we can speed things
// up if so, but not enrolling any BKD cell whose true best distance is > bottom of the current hit queue
- if (pointLat >= minLat && pointLat <= maxLat && pointLon >= minLon && pointLon <= minLon) {
+ if (pointLat >= minLat && pointLat <= maxLat && pointLon >= minLon && pointLon <= maxLon) {
// point is inside the cell!
return 0.0;
}
[4/7] lucene-solr:jira/SOLR-8908: LUCENE-7210: Make TestCore*Parser's
analyzer choice override-able. (Christine Poerschke, Daniel Collins)
Posted by th...@apache.org.
LUCENE-7210: Make TestCore*Parser's analyzer choice override-able. (Christine Poerschke, Daniel Collins)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8d5d2013
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8d5d2013
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8d5d2013
Branch: refs/heads/jira/SOLR-8908
Commit: 8d5d2013431608f397cc7f5cd6be682123246983
Parents: f6c7fc7
Author: Christine Poerschke <cp...@apache.org>
Authored: Thu Apr 14 15:37:41 2016 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Thu Apr 14 16:03:11 2016 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../xml/CoreParserTestIndexData.java | 74 ++++++++++++++++
.../lucene/queryparser/xml/TestCoreParser.java | 89 +++++++++-----------
.../xml/TestCorePlusExtensionsParser.java | 17 +---
.../xml/TestCorePlusQueriesParser.java | 17 +---
5 files changed, 124 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8d5d2013/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index b5cdc7c..decb5db 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -91,6 +91,9 @@ Other
* LUCENE-7205: Remove repeated nl.getLength() calls in
(Boolean|DisjunctionMax|FuzzyLikeThis)QueryBuilder. (Christine Poerschke)
+* LUCENE-7210: Make TestCore*Parser's analyzer choice override-able
+ (Christine Poerschke, Daniel Collins)
+
======================= Lucene 6.0.0 =======================
System Requirements
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8d5d2013/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java
----------------------------------------------------------------------
diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java
new file mode 100644
index 0000000..71b627e
--- /dev/null
+++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/CoreParserTestIndexData.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.queryparser.xml;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.LegacyIntField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.io.BufferedReader;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+
+class CoreParserTestIndexData implements Closeable {
+
+ final Directory dir;
+ final IndexReader reader;
+ final IndexSearcher searcher;
+
+ CoreParserTestIndexData(Analyzer analyzer) throws Exception {
+ BufferedReader d = new BufferedReader(new InputStreamReader(
+ TestCoreParser.class.getResourceAsStream("reuters21578.txt"), StandardCharsets.US_ASCII));
+ dir = LuceneTestCase.newDirectory();
+ IndexWriter writer = new IndexWriter(dir, LuceneTestCase.newIndexWriterConfig(analyzer));
+ String line = d.readLine();
+ while (line != null) {
+ int endOfDate = line.indexOf('\t');
+ String date = line.substring(0, endOfDate).trim();
+ String content = line.substring(endOfDate).trim();
+ Document doc = new Document();
+ doc.add(LuceneTestCase.newTextField("date", date, Field.Store.YES));
+ doc.add(LuceneTestCase.newTextField("contents", content, Field.Store.YES));
+ doc.add(new LegacyIntField("date2", Integer.valueOf(date), Field.Store.NO));
+ doc.add(new IntPoint("date3", Integer.valueOf(date)));
+ writer.addDocument(doc);
+ line = d.readLine();
+ }
+ d.close();
+ writer.close();
+ reader = DirectoryReader.open(dir);
+ searcher = LuceneTestCase.newSearcher(reader, false);
+ }
+
+ @Override
+ public void close() throws IOException {
+ reader.close();
+ dir.close();
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8d5d2013/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java
----------------------------------------------------------------------
diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java
index 04faa7d..9b18c81 100644
--- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java
+++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java
@@ -21,78 +21,41 @@ import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.IntPoint;
-import org.apache.lucene.document.LegacyIntField;
-import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.search.DisjunctionMaxQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-
public class TestCoreParser extends LuceneTestCase {
final private static String defaultField = "contents";
+
private static Analyzer analyzer;
private static CoreParser coreParser;
- private static Directory dir;
- private static IndexReader reader;
- private static IndexSearcher searcher;
- @BeforeClass
- public static void beforeClass() throws Exception {
+ private static CoreParserTestIndexData indexData;
+
+ protected Analyzer newAnalyzer() {
// TODO: rewrite test (this needs to set QueryParser.enablePositionIncrements, too, for work with CURRENT):
- analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
- //initialize the parser
- coreParser = new CoreParser(defaultField, analyzer);
-
- BufferedReader d = new BufferedReader(new InputStreamReader(
- TestCoreParser.class.getResourceAsStream("reuters21578.txt"), StandardCharsets.US_ASCII));
- dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(analyzer));
- String line = d.readLine();
- while (line != null) {
- int endOfDate = line.indexOf('\t');
- String date = line.substring(0, endOfDate).trim();
- String content = line.substring(endOfDate).trim();
- Document doc = new Document();
- doc.add(newTextField("date", date, Field.Store.YES));
- doc.add(newTextField("contents", content, Field.Store.YES));
- doc.add(new LegacyIntField("date2", Integer.valueOf(date), Field.Store.NO));
- doc.add(new IntPoint("date3", Integer.valueOf(date)));
- writer.addDocument(doc);
- line = d.readLine();
- }
- d.close();
- writer.close();
- reader = DirectoryReader.open(dir);
- searcher = newSearcher(reader, false);
+ return new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
+ }
+ protected CoreParser newCoreParser(String defaultField, Analyzer analyzer) {
+ return new CoreParser(defaultField, analyzer);
}
@AfterClass
public static void afterClass() throws Exception {
- reader.close();
- dir.close();
- reader = null;
- searcher = null;
- dir = null;
- coreParser = null;
- analyzer = null;
+ if (indexData != null) {
+ indexData.close();
+ }
}
public void testTermQueryXML() throws ParserException, IOException {
@@ -133,7 +96,7 @@ public class TestCoreParser extends LuceneTestCase {
public void testCustomFieldUserQueryXML() throws ParserException, IOException {
Query q = parse("UserInputQueryCustomField.xml");
- int h = searcher.search(q, 1000).totalHits;
+ int h = searcher().search(q, 1000).totalHits;
assertEquals("UserInputQueryCustomField should produce 0 result ", 0, h);
}
@@ -179,13 +142,38 @@ public class TestCoreParser extends LuceneTestCase {
}
protected Analyzer analyzer() {
+ if (analyzer == null) {
+ analyzer = newAnalyzer();
+ }
return analyzer;
}
protected CoreParser coreParser() {
+ if (coreParser == null) {
+ coreParser = newCoreParser(defaultField, analyzer());
+ }
return coreParser;
}
+ private CoreParserTestIndexData indexData() {
+ if (indexData == null) {
+ try {
+ indexData = new CoreParserTestIndexData(analyzer());
+ } catch (Exception e) {
+ fail("caught Exception "+e);
+ }
+ }
+ return indexData;
+ }
+
+ protected IndexReader reader() {
+ return indexData().reader;
+ }
+
+ protected IndexSearcher searcher() {
+ return indexData().searcher;
+ }
+
protected Query parse(String xmlFileName) throws ParserException, IOException {
try (InputStream xmlStream = TestCoreParser.class.getResourceAsStream(xmlFileName)) {
assertNotNull("Test XML file " + xmlFileName + " cannot be found", xmlStream);
@@ -195,13 +183,14 @@ public class TestCoreParser extends LuceneTestCase {
}
protected Query rewrite(Query q) throws IOException {
- return q.rewrite(reader);
+ return q.rewrite(reader());
}
protected void dumpResults(String qType, Query q, int numDocs) throws IOException {
if (VERBOSE) {
System.out.println("TEST: qType=" + qType + " query=" + q + " numDocs=" + numDocs);
}
+ final IndexSearcher searcher = searcher();
TopDocs hits = searcher.search(q, numDocs);
assertTrue(qType + " should produce results ", hits.totalHits > 0);
if (VERBOSE) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8d5d2013/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusExtensionsParser.java
----------------------------------------------------------------------
diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusExtensionsParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusExtensionsParser.java
index 23cfb50..35f28ef 100644
--- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusExtensionsParser.java
+++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusExtensionsParser.java
@@ -16,11 +16,14 @@
*/
package org.apache.lucene.queryparser.xml;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.search.Query;
public class TestCorePlusExtensionsParser extends TestCorePlusQueriesParser {
- private CoreParser corePlusExtensionsParser;
+ protected CoreParser newCoreParser(String defaultField, Analyzer analyzer) {
+ return new CorePlusExtensionsParser(defaultField, analyzer);
+ }
public void testFuzzyLikeThisQueryXML() throws Exception {
Query q = parse("FuzzyLikeThisQuery.xml");
@@ -31,16 +34,4 @@ public class TestCorePlusExtensionsParser extends TestCorePlusQueriesParser {
dumpResults("FuzzyLikeThis", q, 5);
}
- //================= Helper methods ===================================
-
- @Override
- protected CoreParser coreParser() {
- if (corePlusExtensionsParser == null) {
- corePlusExtensionsParser = new CorePlusExtensionsParser(
- super.defaultField(),
- super.analyzer());
- }
- return corePlusExtensionsParser;
- }
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8d5d2013/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusQueriesParser.java
----------------------------------------------------------------------
diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusQueriesParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusQueriesParser.java
index d87d40b..7e58c47 100644
--- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusQueriesParser.java
+++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCorePlusQueriesParser.java
@@ -16,11 +16,14 @@
*/
package org.apache.lucene.queryparser.xml;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.search.Query;
public class TestCorePlusQueriesParser extends TestCoreParser {
- private CoreParser corePlusQueriesParser;
+ protected CoreParser newCoreParser(String defaultField, Analyzer analyzer) {
+ return new CorePlusQueriesParser(defaultField, analyzer);
+ }
public void testLikeThisQueryXML() throws Exception {
Query q = parse("LikeThisQuery.xml");
@@ -32,16 +35,4 @@ public class TestCorePlusQueriesParser extends TestCoreParser {
dumpResults("boosting ", q, 5);
}
- //================= Helper methods ===================================
-
- @Override
- protected CoreParser coreParser() {
- if (corePlusQueriesParser == null) {
- corePlusQueriesParser = new CorePlusQueriesParser(
- super.defaultField(),
- super.analyzer());
- }
- return corePlusQueriesParser;
- }
-
}
[3/7] lucene-solr:jira/SOLR-8908: LUCENE-7069: add
LatLonPoint.nearest to find N nearest points
Posted by th...@apache.org.
LUCENE-7069: add LatLonPoint.nearest to find N nearest points
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f6c7fc7a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f6c7fc7a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f6c7fc7a
Branch: refs/heads/jira/SOLR-8908
Commit: f6c7fc7a26584c92a81b3a6cbca179ca232808a9
Parents: af5742c
Author: Mike McCandless <mi...@apache.org>
Authored: Thu Apr 14 11:02:25 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Thu Apr 14 11:02:25 2016 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../codecs/lucene60/Lucene60PointsReader.java | 5 +-
.../org/apache/lucene/util/bkd/BKDReader.java | 47 ++-
.../org/apache/lucene/document/LatLonPoint.java | 57 +++-
.../apache/lucene/document/NearestNeighbor.java | 316 +++++++++++++++++++
.../org/apache/lucene/document/TestNearest.java | 245 ++++++++++++++
6 files changed, 657 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6c7fc7a/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 5cd5d04..b5cdc7c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -16,6 +16,9 @@ New Features
* LUCENE-7140: Add PlanetModel.bisection to spatial3d (Karl Wright via
Mike McCandless)
+* LUCENE-7069: Add LatLonPoint.nearest, to find nearest N points to a
+ provided query point (Mike McCandless)
+
API Changes
* LUCENE-7184: Refactor LatLonPoint encoding methods to new GeoEncodingUtils
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6c7fc7a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsReader.java
index e7a612c..8c91a99 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsReader.java
@@ -114,7 +114,10 @@ public class Lucene60PointsReader extends PointsReader implements Closeable {
}
}
- private BKDReader getBKDReader(String fieldName) {
+ /** Returns the underlying {@link BKDReader}.
+ *
+ * @lucene.internal */
+ public BKDReader getBKDReader(String fieldName) {
FieldInfo fieldInfo = readState.fieldInfos.fieldInfo(fieldName);
if (fieldInfo == null) {
throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6c7fc7a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
index b904a50..9a123a2 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
@@ -205,12 +205,7 @@ public class BKDReader implements Accountable {
byte[] rootMinPacked = new byte[packedBytesLength];
byte[] rootMaxPacked = new byte[packedBytesLength];
Arrays.fill(rootMaxPacked, (byte) 0xff);
-
- IntersectState state = new IntersectState(in.clone(), numDims, packedBytesLength,
- maxPointsInLeafNode,
- new VerifyVisitor(numDims, bytesPerDim, maxDoc));
-
- verify(state, 1, rootMinPacked, rootMaxPacked);
+ verify(getIntersectState(new VerifyVisitor(numDims, bytesPerDim, maxDoc)), 1, rootMinPacked, rootMaxPacked);
}
private void verify(IntersectState state, int nodeID, byte[] cellMinPacked, byte[] cellMaxPacked) throws IOException {
@@ -258,7 +253,8 @@ public class BKDReader implements Accountable {
}
}
- static final class IntersectState {
+ /** Used to track all state for a single call to {@link #intersect}. */
+ public static final class IntersectState {
final IndexInput in;
final int[] scratchDocIDs;
final byte[] scratchPackedValue;
@@ -279,11 +275,7 @@ public class BKDReader implements Accountable {
}
public void intersect(IntersectVisitor visitor) throws IOException {
- IntersectState state = new IntersectState(in.clone(), numDims,
- packedBytesLength,
- maxPointsInLeafNode,
- visitor);
- intersect(state, 1, minPackedValue, maxPackedValue);
+ intersect(getIntersectState(visitor), 1, minPackedValue, maxPackedValue);
}
/** Fast path: this is called when the query box fully encompasses all cells under this node. */
@@ -300,6 +292,25 @@ public class BKDReader implements Accountable {
}
}
+ /** Create a new {@link IntersectState} */
+ public IntersectState getIntersectState(IntersectVisitor visitor) {
+ return new IntersectState(in.clone(), numDims,
+ packedBytesLength,
+ maxPointsInLeafNode,
+ visitor);
+ }
+
+ /** Visits all docIDs and packed values in a single leaf block */
+ public void visitLeafBlockValues(int nodeID, IntersectState state) throws IOException {
+ int leafID = nodeID - leafNodeOffset;
+
+ // Leaf node; scan and filter all points in this block:
+ int count = readDocIDs(state.in, leafBlockFPs[leafID], state.scratchDocIDs);
+
+ // Again, this time reading values and checking with the visitor
+ visitDocValues(state.commonPrefixLengths, state.scratchPackedValue, state.in, state.scratchDocIDs, count, state.visitor);
+ }
+
protected void visitDocIDs(IndexInput in, long blockFP, IntersectVisitor visitor) throws IOException {
// Leaf node
in.seek(blockFP);
@@ -414,6 +425,14 @@ public class BKDReader implements Accountable {
}
}
+ /** Copies the split value for this node into the provided byte array */
+ public void copySplitValue(int nodeID, byte[] splitPackedValue) {
+ int address = nodeID * (bytesPerDim+1);
+ int splitDim = splitPackedValues[address] & 0xff;
+ assert splitDim < numDims;
+ System.arraycopy(splitPackedValues, address+1, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
+ }
+
@Override
public long ramBytesUsed() {
return splitPackedValues.length +
@@ -443,4 +462,8 @@ public class BKDReader implements Accountable {
public int getDocCount() {
return docCount;
}
+
+ public boolean isLeafNode(int nodeID) {
+ return nodeID >= leafNodeOffset;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6c7fc7a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
index b1e0b44..06aefde 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
@@ -16,20 +16,33 @@
*/
package org.apache.lucene.document;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.NumericUtils;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.lucene.codecs.lucene60.Lucene60PointsFormat;
+import org.apache.lucene.codecs.lucene60.Lucene60PointsReader;
+import org.apache.lucene.geo.Polygon;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.PointRangeQuery;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.SortField;
-import org.apache.lucene.geo.Polygon;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.bkd.BKDReader;
import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude;
import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude;
@@ -289,4 +302,42 @@ public class LatLonPoint extends Field {
public static SortField newDistanceSort(String field, double latitude, double longitude) {
return new LatLonPointSortField(field, latitude, longitude);
}
+
+ /**
+ * Finds the {@code topN} nearest indexed points to the provided point, according to Haversine distance.
+ * This is functionally equivalent to running {@link MatchAllDocsQuery} with a {@link #newDistanceSort},
+ * but is far more efficient since it takes advantage of properties the indexed BKD tree. Currently this
+ * only works with {@link Lucene60PointsFormat} (used by the default codec).
+ */
+ public static TopFieldDocs nearest(IndexSearcher s, String fieldName, double latitude, double longitude, int n) throws IOException {
+ List<BKDReader> readers = new ArrayList<>();
+ List<Integer> docBases = new ArrayList<>();
+ List<Bits> liveDocs = new ArrayList<>();
+ int totalHits = 0;
+ for(LeafReaderContext leaf : s.getIndexReader().leaves()) {
+ PointValues points = leaf.reader().getPointValues();
+ if (points != null) {
+ if (points instanceof Lucene60PointsReader == false) {
+ throw new IllegalArgumentException("can only run on Lucene60PointsReader points implementation, but got " + points);
+ }
+ totalHits += points.getDocCount(fieldName);
+ BKDReader reader = ((Lucene60PointsReader) points).getBKDReader(fieldName);
+ if (reader != null) {
+ readers.add(reader);
+ docBases.add(leaf.docBase);
+ liveDocs.add(leaf.reader().getLiveDocs());
+ }
+ }
+ }
+
+ NearestNeighbor.NearestHit[] hits = NearestNeighbor.nearest(latitude, longitude, readers, liveDocs, docBases, n);
+
+ // Convert to TopFieldDocs:
+ ScoreDoc[] scoreDocs = new ScoreDoc[hits.length];
+ for(int i=0;i<hits.length;i++) {
+ NearestNeighbor.NearestHit hit = hits[i];
+ scoreDocs[i] = new FieldDoc(hit.docID, 0.0f, new Object[] {Double.valueOf(hit.distanceMeters)});
+ }
+ return new TopFieldDocs(totalHits, scoreDocs, null, 0.0f);
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6c7fc7a/lucene/sandbox/src/java/org/apache/lucene/document/NearestNeighbor.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/NearestNeighbor.java b/lucene/sandbox/src/java/org/apache/lucene/document/NearestNeighbor.java
new file mode 100644
index 0000000..581be46
--- /dev/null
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/NearestNeighbor.java
@@ -0,0 +1,316 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.document;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+import org.apache.lucene.geo.Rectangle;
+import org.apache.lucene.index.PointValues.IntersectVisitor;
+import org.apache.lucene.index.PointValues.Relation;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.SloppyMath;
+import org.apache.lucene.util.bkd.BKDReader;
+
+import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude;
+import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude;
+
+/**
+ * KNN search on top of 2D lat/lon indexed points.
+ *
+ * @lucene.experimental
+ */
+class NearestNeighbor {
+
+ static class Cell implements Comparable<Cell> {
+ final int readerIndex;
+ final int nodeID;
+ final byte[] minPacked;
+ final byte[] maxPacked;
+
+ /** The closest possible distance of all points in this cell */
+ final double distanceMeters;
+
+ public Cell(int readerIndex, int nodeID, byte[] minPacked, byte[] maxPacked, double distanceMeters) {
+ this.readerIndex = readerIndex;
+ this.nodeID = nodeID;
+ this.minPacked = minPacked.clone();
+ this.maxPacked = maxPacked.clone();
+ this.distanceMeters = distanceMeters;
+ }
+
+ public int compareTo(Cell other) {
+ return Double.compare(distanceMeters, other.distanceMeters);
+ }
+
+ @Override
+ public String toString() {
+ double minLat = decodeLatitude(minPacked, 0);
+ double minLon = decodeLongitude(minPacked, Integer.BYTES);
+ double maxLat = decodeLatitude(maxPacked, 0);
+ double maxLon = decodeLongitude(maxPacked, Integer.BYTES);
+ return "Cell(readerIndex=" + readerIndex + " lat=" + minLat + " TO " + maxLat + ", lon=" + minLon + " TO " + maxLon + ")";
+ }
+ }
+
+ private static class NearestVisitor implements IntersectVisitor {
+
+ public int curDocBase;
+ public Bits curLiveDocs;
+ final int topN;
+ final PriorityQueue<NearestHit> hitQueue;
+ final double pointLat;
+ final double pointLon;
+ private int setBottomCounter;
+
+ private double minLon = Double.NEGATIVE_INFINITY;
+ private double maxLon = Double.POSITIVE_INFINITY;
+ private double minLat = Double.NEGATIVE_INFINITY;
+ private double maxLat = Double.POSITIVE_INFINITY;
+
+ // second set of longitude ranges to check (for cross-dateline case)
+ private double minLon2 = Double.POSITIVE_INFINITY;
+
+ int pointCheckCount;
+
+ public NearestVisitor(PriorityQueue<NearestHit> hitQueue, int topN, double pointLat, double pointLon) {
+ this.hitQueue = hitQueue;
+ this.topN = topN;
+ this.pointLat = pointLat;
+ this.pointLon = pointLon;
+ }
+
+ @Override
+ public void visit(int docID) {
+ throw new AssertionError();
+ }
+
+ private void maybeUpdateBBox() {
+ if (setBottomCounter < 1024 || (setBottomCounter & 0x3F) == 0x3F) {
+ NearestHit hit = hitQueue.peek();
+ Rectangle box = Rectangle.fromPointDistance(pointLat, pointLon, hit.distanceMeters);
+ minLat = box.minLat;
+ maxLat = box.maxLat;
+ if (box.crossesDateline()) {
+ // box1
+ minLon = Double.NEGATIVE_INFINITY;
+ maxLon = box.maxLon;
+ // box2
+ minLon2 = box.minLon;
+ } else {
+ minLon = box.minLon;
+ maxLon = box.maxLon;
+ // disable box2
+ minLon2 = Double.POSITIVE_INFINITY;
+ }
+ }
+ setBottomCounter++;
+ }
+
+ @Override
+ public void visit(int docID, byte[] packedValue) {
+ //System.out.println("visit docID=" + docID + " liveDocs=" + curLiveDocs);
+
+ if (curLiveDocs != null && curLiveDocs.get(docID) == false) {
+ return;
+ }
+
+ // TODO: work in int space, use haversinSortKey
+
+ double docLatitude = decodeLatitude(packedValue, 0);
+ double docLongitude = decodeLongitude(packedValue, Integer.BYTES);
+
+ // test bounding box
+ if (docLatitude < minLat || docLatitude > maxLat) {
+ return;
+ }
+ if ((docLongitude < minLon || docLongitude > maxLon) && (docLongitude < minLon2)) {
+ return;
+ }
+
+ pointCheckCount++;
+
+ double distanceMeters = SloppyMath.haversinMeters(pointLat, pointLon, docLatitude, docLongitude);
+
+ //System.out.println(" visit docID=" + docID + " distanceMeters=" + distanceMeters + " docLat=" + docLatitude + " docLon=" + docLongitude);
+
+ int fullDocID = curDocBase + docID;
+
+ if (hitQueue.size() == topN) {
+ // queue already full
+ NearestHit hit = hitQueue.peek();
+ //System.out.println(" bottom distanceMeters=" + hit.distanceMeters);
+ // we don't collect docs in order here, so we must also test the tie-break case ourselves:
+ if (distanceMeters < hit.distanceMeters || (distanceMeters == hit.distanceMeters && fullDocID < hit.docID)) {
+ hitQueue.poll();
+ hit.docID = fullDocID;
+ hit.distanceMeters = distanceMeters;
+ hitQueue.offer(hit);
+ //System.out.println(" ** keep2, now bottom=" + hit);
+ maybeUpdateBBox();
+ }
+
+ } else {
+ NearestHit hit = new NearestHit();
+ hit.docID = fullDocID;
+ hit.distanceMeters = distanceMeters;
+ hitQueue.offer(hit);
+ //System.out.println(" ** keep1, now bottom=" + hit);
+ }
+ }
+
+ @Override
+ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
+ throw new AssertionError();
+ }
+ }
+
+ /** Holds one hit from {@link LatLonPoint#nearest} */
+ static class NearestHit {
+ public int docID;
+ public double distanceMeters;
+
+ @Override
+ public String toString() {
+ return "NearestHit(docID=" + docID + " distanceMeters=" + distanceMeters + ")";
+ }
+ }
+
+ // TODO: can we somehow share more with, or simply directly use, the LatLonPointDistanceComparator? It's really doing the same thing as
+ // our hitQueue...
+
+ public static NearestHit[] nearest(double pointLat, double pointLon, List<BKDReader> readers, List<Bits> liveDocs, List<Integer> docBases, final int n) throws IOException {
+
+ //System.out.println("NEAREST: r=" + r + " liveDocs=" + liveDocs);
+ // Holds closest collected points seen so far:
+ // TODO: if we used lucene's PQ we could just updateTop instead of poll/offer:
+ final PriorityQueue<NearestHit> hitQueue = new PriorityQueue<>(n, new Comparator<NearestHit>() {
+ @Override
+ public int compare(NearestHit a, NearestHit b) {
+ // sort by opposite distanceMeters natural order
+ int cmp = Double.compare(a.distanceMeters, b.distanceMeters);
+ if (cmp != 0) {
+ return -cmp;
+ }
+
+ // tie-break by higher docID:
+ return b.docID - a.docID;
+ }
+ });
+
+ // Holds all cells, sorted by closest to the point:
+ PriorityQueue<Cell> cellQueue = new PriorityQueue<>();
+
+ NearestVisitor visitor = new NearestVisitor(hitQueue, n, pointLat, pointLon);
+ List<BKDReader.IntersectState> states = new ArrayList<>();
+
+ // Add root cell for each reader into the queue:
+ for(int i=0;i<readers.size();i++) {
+ BKDReader reader = readers.get(i);
+ byte[] minPackedValue = reader.getMinPackedValue();
+ double minLat = decodeLatitude(minPackedValue, 0);
+ double minLon = decodeLongitude(minPackedValue, Integer.BYTES);
+
+ byte[] maxPackedValue = reader.getMaxPackedValue();
+ double maxLat = decodeLatitude(maxPackedValue, 0);
+ double maxLon = decodeLongitude(maxPackedValue, Integer.BYTES);
+ states.add(reader.getIntersectState(visitor));
+
+ cellQueue.offer(new Cell(i, 1, reader.getMinPackedValue(), reader.getMaxPackedValue(),
+ approxBestDistance(minLat, maxLat, minLon, maxLon, pointLat, pointLon)));
+ }
+
+ while (cellQueue.size() > 0) {
+ Cell cell = cellQueue.poll();
+ //System.out.println(" visit " + cell);
+
+ // TODO: if we replace approxBestDistance with actualBestDistance, we can put an opto here to break once this "best" cell is fully outside of the hitQueue bottom's radius:
+ BKDReader reader = readers.get(cell.readerIndex);
+
+ if (reader.isLeafNode(cell.nodeID)) {
+ //System.out.println(" leaf");
+ // Leaf block: visit all points and possibly collect them:
+ visitor.curDocBase = docBases.get(cell.readerIndex);
+ visitor.curLiveDocs = liveDocs.get(cell.readerIndex);
+ reader.visitLeafBlockValues(cell.nodeID, states.get(cell.readerIndex));
+ //System.out.println(" now " + hitQueue.size() + " hits");
+ } else {
+ //System.out.println(" non-leaf");
+ // Non-leaf block: split into two cells and put them back into the queue:
+
+ double cellMinLat = decodeLatitude(cell.minPacked, 0);
+ double cellMinLon = decodeLongitude(cell.minPacked, Integer.BYTES);
+ double cellMaxLat = decodeLatitude(cell.maxPacked, 0);
+ double cellMaxLon = decodeLongitude(cell.maxPacked, Integer.BYTES);
+
+ if (cellMaxLat < visitor.minLat || visitor.maxLat < cellMinLat || ((cellMaxLon < visitor.minLon || visitor.maxLon < cellMinLon) && cellMaxLon < visitor.minLon2)) {
+ // this cell is outside our search bbox; don't bother exploring any more
+ continue;
+ }
+
+ byte[] splitPackedValue = cell.maxPacked.clone();
+ reader.copySplitValue(cell.nodeID, splitPackedValue);
+ cellQueue.offer(new Cell(cell.readerIndex, 2*cell.nodeID, cell.minPacked, splitPackedValue,
+ approxBestDistance(cell.minPacked, splitPackedValue, pointLat, pointLon)));
+
+ splitPackedValue = cell.minPacked.clone();
+ reader.copySplitValue(cell.nodeID, splitPackedValue);
+ cellQueue.offer(new Cell(cell.readerIndex, 2*cell.nodeID+1, splitPackedValue, cell.maxPacked,
+ approxBestDistance(splitPackedValue, cell.maxPacked, pointLat, pointLon)));
+ }
+ }
+
+ NearestHit[] hits = new NearestHit[hitQueue.size()];
+ int downTo = hitQueue.size()-1;
+ while (hitQueue.size() != 0) {
+ hits[downTo] = hitQueue.poll();
+ downTo--;
+ }
+
+ return hits;
+ }
+
+ // NOTE: incoming args never cross the dateline, since they are a BKD cell
+ private static double approxBestDistance(byte[] minPackedValue, byte[] maxPackedValue, double pointLat, double pointLon) {
+ double minLat = decodeLatitude(minPackedValue, 0);
+ double minLon = decodeLongitude(minPackedValue, Integer.BYTES);
+ double maxLat = decodeLatitude(maxPackedValue, 0);
+ double maxLon = decodeLongitude(maxPackedValue, Integer.BYTES);
+ return approxBestDistance(minLat, maxLat, minLon, maxLon, pointLat, pointLon);
+ }
+
+ // NOTE: incoming args never cross the dateline, since they are a BKD cell
+ private static double approxBestDistance(double minLat, double maxLat, double minLon, double maxLon, double pointLat, double pointLon) {
+
+ // TODO: can we make this the trueBestDistance? I.e., minimum distance between the point and ANY point on the box? we can speed things
+ // up if so, but not enrolling any BKD cell whose true best distance is > bottom of the current hit queue
+
+ if (pointLat >= minLat && pointLat <= maxLat && pointLon >= minLon && pointLon <= minLon) {
+ // point is inside the cell!
+ return 0.0;
+ }
+
+ double d1 = SloppyMath.haversinMeters(pointLat, pointLon, minLat, minLon);
+ double d2 = SloppyMath.haversinMeters(pointLat, pointLon, minLat, maxLon);
+ double d3 = SloppyMath.haversinMeters(pointLat, pointLon, maxLat, maxLon);
+ double d4 = SloppyMath.haversinMeters(pointLat, pointLon, maxLat, minLon);
+ return Math.min(Math.min(d1, d2), Math.min(d3, d4));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6c7fc7a/lucene/sandbox/src/test/org/apache/lucene/document/TestNearest.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestNearest.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestNearest.java
new file mode 100644
index 0000000..0e3044e
--- /dev/null
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestNearest.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.document;
+
+import java.util.Arrays;
+import java.util.Comparator;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.document.NearestNeighbor.NearestHit;
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.apache.lucene.geo.GeoTestUtil;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.SerialMergeScheduler;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.SloppyMath;
+import org.apache.lucene.util.TestUtil;
+
+public class TestNearest extends LuceneTestCase {
+
+ public void testNearestNeighborWithDeletedDocs() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, getIndexWriterConfig());
+ Document doc = new Document();
+ doc.add(new LatLonPoint("point", 40.0, 50.0));
+ doc.add(new StringField("id", "0", Field.Store.YES));
+ w.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new LatLonPoint("point", 45.0, 55.0));
+ doc.add(new StringField("id", "1", Field.Store.YES));
+ w.addDocument(doc);
+
+ DirectoryReader r = w.getReader();
+ IndexSearcher s = newSearcher(r);
+ FieldDoc hit = (FieldDoc) LatLonPoint.nearest(s, "point", 40.0, 50.0, 1).scoreDocs[0];
+ assertEquals("0", r.document(hit.doc).getField("id").stringValue());
+ r.close();
+
+ w.deleteDocuments(new Term("id", "0"));
+ r = w.getReader();
+ s = newSearcher(r);
+ hit = (FieldDoc) LatLonPoint.nearest(s, "point", 40.0, 50.0, 1).scoreDocs[0];
+ assertEquals("1", r.document(hit.doc).getField("id").stringValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testNearestNeighborWithAllDeletedDocs() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, getIndexWriterConfig());
+ Document doc = new Document();
+ doc.add(new LatLonPoint("point", 40.0, 50.0));
+ doc.add(new StringField("id", "0", Field.Store.YES));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(new LatLonPoint("point", 45.0, 55.0));
+ doc.add(new StringField("id", "1", Field.Store.YES));
+ w.addDocument(doc);
+
+ DirectoryReader r = w.getReader();
+ IndexSearcher s = newSearcher(r);
+ FieldDoc hit = (FieldDoc) LatLonPoint.nearest(s, "point", 40.0, 50.0, 1).scoreDocs[0];
+ assertEquals("0", r.document(hit.doc).getField("id").stringValue());
+ r.close();
+
+ w.deleteDocuments(new Term("id", "0"));
+ w.deleteDocuments(new Term("id", "1"));
+ r = w.getReader();
+ s = newSearcher(r);
+ assertEquals(0, LatLonPoint.nearest(s, "point", 40.0, 50.0, 1).scoreDocs.length);
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testTieBreakByDocID() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, getIndexWriterConfig());
+ Document doc = new Document();
+ doc.add(new LatLonPoint("point", 40.0, 50.0));
+ doc.add(new StringField("id", "0", Field.Store.YES));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(new LatLonPoint("point", 40.0, 50.0));
+ doc.add(new StringField("id", "1", Field.Store.YES));
+ w.addDocument(doc);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ ScoreDoc[] hits = LatLonPoint.nearest(newSearcher(r), "point", 45.0, 50.0, 2).scoreDocs;
+ assertEquals("0", r.document(hits[0].doc).getField("id").stringValue());
+ assertEquals("1", r.document(hits[1].doc).getField("id").stringValue());
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testNearestNeighborWithNoDocs() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, getIndexWriterConfig());
+ DirectoryReader r = w.getReader();
+ assertEquals(0, LatLonPoint.nearest(newSearcher(r), "point", 40.0, 50.0, 1).scoreDocs.length);
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ private double quantizeLat(double latRaw) {
+ return GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(latRaw));
+ }
+
+ private double quantizeLon(double lonRaw) {
+ return GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lonRaw));
+ }
+
+ public void testNearestNeighborRandom() throws Exception {
+
+ int numPoints = atLeast(5000);
+ Directory dir;
+ if (numPoints > 100000) {
+ dir = newFSDirectory(createTempDir(getClass().getSimpleName()));
+ } else {
+ dir = newDirectory();
+ }
+ double[] lats = new double[numPoints];
+ double[] lons = new double[numPoints];
+
+ IndexWriterConfig iwc = getIndexWriterConfig();
+ iwc.setMergePolicy(newLogMergePolicy());
+ iwc.setMergeScheduler(new SerialMergeScheduler());
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+ for(int id=0;id<numPoints;id++) {
+ lats[id] = quantizeLat(GeoTestUtil.nextLatitude());
+ lons[id] = quantizeLon(GeoTestUtil.nextLongitude());
+ Document doc = new Document();
+ doc.add(new LatLonPoint("point", lats[id], lons[id]));
+ doc.add(new StoredField("id", id));
+ w.addDocument(doc);
+ }
+
+ if (random().nextBoolean()) {
+ w.forceMerge(1);
+ }
+
+ DirectoryReader r = w.getReader();
+ if (VERBOSE) {
+ System.out.println("TEST: reader=" + r);
+ }
+ IndexSearcher s = newSearcher(r);
+ int iters = atLeast(100);
+ for(int iter=0;iter<iters;iter++) {
+ if (VERBOSE) {
+ System.out.println("\nTEST: iter=" + iter);
+ }
+ double pointLat = GeoTestUtil.nextLatitude();
+ double pointLon = GeoTestUtil.nextLongitude();
+
+ // dumb brute force search to get the expected result:
+ NearestHit[] expectedHits = new NearestHit[lats.length];
+ for(int id=0;id<lats.length;id++) {
+ NearestHit hit = new NearestHit();
+ hit.distanceMeters = SloppyMath.haversinMeters(pointLat, pointLon, lats[id], lons[id]);
+ hit.docID = id;
+ expectedHits[id] = hit;
+ }
+
+ Arrays.sort(expectedHits, new Comparator<NearestHit>() {
+ @Override
+ public int compare(NearestHit a, NearestHit b) {
+ int cmp = Double.compare(a.distanceMeters, b.distanceMeters);
+ if (cmp != 0) {
+ return cmp;
+ }
+ // tie break by smaller docID:
+ return a.docID - b.docID;
+ }
+ });
+
+ int topN = TestUtil.nextInt(random(), 1, lats.length);
+
+ if (VERBOSE) {
+ System.out.println("\nhits for pointLat=" + pointLat + " pointLon=" + pointLon);
+ }
+
+ // Also test with MatchAllDocsQuery, sorting by distance:
+ TopFieldDocs fieldDocs = s.search(new MatchAllDocsQuery(), topN, new Sort(LatLonPoint.newDistanceSort("point", pointLat, pointLon)));
+
+ ScoreDoc[] hits = LatLonPoint.nearest(s, "point", pointLat, pointLon, topN).scoreDocs;
+ for(int i=0;i<topN;i++) {
+ NearestHit expected = expectedHits[i];
+ FieldDoc expected2 = (FieldDoc) fieldDocs.scoreDocs[i];
+ FieldDoc actual = (FieldDoc) hits[i];
+ Document actualDoc = r.document(actual.doc);
+
+ if (VERBOSE) {
+ System.out.println("hit " + i);
+ System.out.println(" expected id=" + expected.docID + " lat=" + lats[expected.docID] + " lon=" + lons[expected.docID] + " distance=" + expected.distanceMeters + " meters");
+ System.out.println(" actual id=" + actualDoc.getField("id") + " distance=" + actual.fields[0] + " meters");
+ }
+
+ assertEquals(expected.docID, actual.doc);
+ assertEquals(expected.distanceMeters, ((Double) actual.fields[0]).doubleValue(), 0.0);
+
+ assertEquals(expected.docID, expected.docID);
+ assertEquals(((Double) expected2.fields[0]).doubleValue(), expected.distanceMeters, 0.0);
+ }
+ }
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ private IndexWriterConfig getIndexWriterConfig() {
+ IndexWriterConfig iwc = newIndexWriterConfig();
+ iwc.setCodec(Codec.forName("Lucene60"));
+ return iwc;
+ }
+}