You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2016/03/21 01:43:23 UTC
[01/50] lucene-solr:jira/SOLR-445: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Repository: lucene-solr
Updated Branches:
refs/heads/jira/SOLR-445 a0d48f873 -> 21c0fe690
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/459d6fd9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/459d6fd9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/459d6fd9
Branch: refs/heads/jira/SOLR-445
Commit: 459d6fd9f9bec34d689ab50ade60c841bef42a95
Parents: 3bc2b9e e169050
Author: Mike McCandless <mi...@apache.org>
Authored: Sat Mar 12 04:57:02 2016 -0500
Committer: Mike McCandless <mi...@apache.org>
Committed: Sat Mar 12 04:57:02 2016 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 6 +-
.../org/apache/lucene/util/bkd/BKDWriter.java | 15 +-
.../org/apache/lucene/document/LatLonPoint.java | 69 +++++-
.../document/LatLonPointDistanceComparator.java | 212 +++++++++++++++++++
.../lucene/document/LatLonPointSortField.java | 108 ++++++++++
.../apache/lucene/document/TestLatLonPoint.java | 3 +
.../document/TestLatLonPointDistanceSort.java | 190 +++++++++++++++++
solr/CHANGES.txt | 2 +
.../org/apache/solr/update/VersionInfo.java | 8 +-
.../client/solrj/io/stream/DaemonStream.java | 6 +-
.../solrj/io/stream/StreamExpressionTest.java | 8 +-
solr/webapp/web/partials/query.html | 12 +-
12 files changed, 612 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
[40/50] lucene-solr:jira/SOLR-445: SOLR-8860: Remove back-compat
handling of router format made in SOLR-4221 in 4.5.0
Posted by ho...@apache.org.
SOLR-8860: Remove back-compat handling of router format made in SOLR-4221 in 4.5.0
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ae846bfb
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ae846bfb
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ae846bfb
Branch: refs/heads/jira/SOLR-445
Commit: ae846bfb492fd91e30daac017c6587083e278236
Parents: 3cdde08
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Thu Mar 17 09:21:30 2016 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Thu Mar 17 09:21:30 2016 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../org/apache/solr/common/cloud/DocRouter.java | 27 +++++++-------------
2 files changed, 11 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae846bfb/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ffcfabd..b5aa670 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -61,6 +61,8 @@ Other Changes
* SOLR-7516: Improve javadocs for JavaBinCodec, ObjectResolver and enforce the single-usage policy.
(Jason Gerlowski, Benoit Vanalderweireldt, shalin)
+* SOLR-8860: Remove back-compat handling of router format made in SOLR-4221 in 4.5.0. (shalin)
+
================== 6.0.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae846bfb/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java b/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java
index e64c064..6fffb3a 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java
@@ -48,33 +48,24 @@ public abstract class DocRouter {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown document router '"+ routerName + "'");
}
- protected String getRouteField(DocCollection coll){
- if(coll == null) return null;
- Object o = coll.get(DOC_ROUTER);
- if (o instanceof String) {
- return null;
- //old format. cannot have a routefield. Ignore it
- }
- Map m = (Map) o;
- if(m == null) return null;
+ protected String getRouteField(DocCollection coll) {
+ if (coll == null) return null;
+ Map m = (Map) coll.get(DOC_ROUTER);
+ if (m == null) return null;
return (String) m.get("field");
-
}
- public static Map<String,Object> getRouterSpec(ZkNodeProps props){
- Map<String,Object> map = new LinkedHashMap<>();
+ public static Map<String, Object> getRouterSpec(ZkNodeProps props) {
+ Map<String, Object> map = new LinkedHashMap<>();
for (String s : props.keySet()) {
- if(s.startsWith("router.")){
+ if (s.startsWith("router.")) {
map.put(s.substring(7), props.get(s));
}
}
- Object o = props.get("router");
- if (o instanceof String) {
- map.put("name", o);
- } else if (map.get("name") == null) {
+ if (map.get("name") == null) {
map.put("name", DEFAULT_NAME);
}
- return map;
+ return map;
}
// currently just an implementation detail...
[12/50] lucene-solr:jira/SOLR-445: optimize offline -> offline
partition
Posted by ho...@apache.org.
optimize offline -> offline partition
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/983908c8
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/983908c8
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/983908c8
Branch: refs/heads/jira/SOLR-445
Commit: 983908c80989d2af6868c8e1d99925a52d79a65e
Parents: d8eac8e
Author: Mike McCandless <mi...@apache.org>
Authored: Sun Mar 13 08:55:31 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Sun Mar 13 08:55:31 2016 -0400
----------------------------------------------------------------------
.../org/apache/lucene/util/bkd/BKDWriter.java | 19 +----
.../apache/lucene/util/bkd/HeapPointReader.java | 4 +-
.../lucene/util/bkd/OfflinePointReader.java | 77 +++++++++++++++++++-
.../lucene/util/bkd/OfflinePointWriter.java | 2 +-
.../org/apache/lucene/util/bkd/PointReader.java | 37 ++++++++--
5 files changed, 111 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/983908c8/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index d4e30b7..c5cdc30 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -1176,24 +1176,7 @@ public class BKDWriter implements Closeable {
PointWriter rightPointWriter = getPointWriter(source.count - leftCount, "right" + dim);
PointReader reader = slices[dim].writer.getReader(slices[dim].start);) {
- // Partition this source according to how the splitDim split the values:
- long nextRightCount = 0;
- for (long i=0;i<source.count;i++) {
- boolean result = reader.next();
- assert result;
- byte[] packedValue = reader.packedValue();
- long ord = reader.ord();
- int docID = reader.docID();
- if (ordBitSet.get(ord)) {
- rightPointWriter.append(packedValue, ord, docID);
- nextRightCount++;
- if (dim == dimToClear) {
- ordBitSet.clear(ord);
- }
- } else {
- leftPointWriter.append(packedValue, ord, docID);
- }
- }
+ long nextRightCount = reader.split(source.count, ordBitSet, leftPointWriter, rightPointWriter, dim == dimToClear);
leftSlices[dim] = new PathSlice(leftPointWriter, 0, leftCount);
rightSlices[dim] = new PathSlice(rightPointWriter, 0, rightCount);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/983908c8/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
index 63c7869..cd9152e 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
@@ -18,9 +18,7 @@ package org.apache.lucene.util.bkd;
import java.util.List;
-import org.apache.lucene.util.PagedBytes;
-
-final class HeapPointReader implements PointReader {
+final class HeapPointReader extends PointReader {
private int curRead;
final List<byte[]> blocks;
final int valuesPerBlock;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/983908c8/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java
index 3c4b8b5..c8ab47e 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java
@@ -22,9 +22,11 @@ import java.io.IOException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.LongBitSet;
/** Reads points from disk in a fixed-with format, previously written with {@link OfflinePointWriter}. */
-final class OfflinePointReader implements PointReader {
+final class OfflinePointReader extends PointReader {
long countLeft;
private final IndexInput in;
private final byte[] packedValue;
@@ -90,5 +92,78 @@ final class OfflinePointReader implements PointReader {
public void close() throws IOException {
in.close();
}
+
+ @Override
+ public long split(long count, LongBitSet rightTree, PointWriter left, PointWriter right, boolean doClearBits) throws IOException {
+
+ if (left instanceof OfflinePointWriter == false ||
+ right instanceof OfflinePointWriter == false) {
+ return super.split(count, rightTree, left, right, doClearBits);
+ }
+
+ // We specialize the offline -> offline split since the default impl
+ // is somewhat wasteful otherwise (e.g. decoding docID when we don't
+ // need to)
+
+ int packedBytesLength = packedValue.length;
+
+ int bytesPerDoc = packedBytesLength + Integer.BYTES;
+ if (longOrds) {
+ bytesPerDoc += Long.BYTES;
+ } else {
+ bytesPerDoc += Integer.BYTES;
+ }
+
+ long rightCount = 0;
+
+ IndexOutput rightOut = ((OfflinePointWriter) right).out;
+ IndexOutput leftOut = ((OfflinePointWriter) left).out;
+
+ ((OfflinePointWriter) right).count = count;
+ ((OfflinePointWriter) left).count = count;
+
+ assert count <= countLeft: "count=" + count + " countLeft=" + countLeft;
+
+ countLeft -= count;
+
+ byte[] buffer = new byte[bytesPerDoc];
+ while (count > 0) {
+ in.readBytes(buffer, 0, buffer.length);
+ long ord;
+ if (longOrds) {
+ ord = readLong(buffer, packedBytesLength);
+ } else {
+ ord = readInt(buffer, packedBytesLength);
+ }
+ if (rightTree.get(ord)) {
+ rightOut.writeBytes(buffer, 0, bytesPerDoc);
+ if (doClearBits) {
+ rightTree.clear(ord);
+ }
+ rightCount++;
+ } else {
+ leftOut.writeBytes(buffer, 0, bytesPerDoc);
+ }
+
+ count--;
+ }
+
+ return rightCount;
+ }
+
+ // Poached from ByteArrayDataInput:
+ private static long readLong(byte[] bytes, int pos) {
+ final int i1 = ((bytes[pos++] & 0xff) << 24) | ((bytes[pos++] & 0xff) << 16) |
+ ((bytes[pos++] & 0xff) << 8) | (bytes[pos++] & 0xff);
+ final int i2 = ((bytes[pos++] & 0xff) << 24) | ((bytes[pos++] & 0xff) << 16) |
+ ((bytes[pos++] & 0xff) << 8) | (bytes[pos++] & 0xff);
+ return (((long)i1) << 32) | (i2 & 0xFFFFFFFFL);
+ }
+
+ // Poached from ByteArrayDataInput:
+ private static int readInt(byte[] bytes, int pos) {
+ return ((bytes[pos++] & 0xFF) << 24) | ((bytes[pos++] & 0xFF) << 16)
+ | ((bytes[pos++] & 0xFF) << 8) | (bytes[pos++] & 0xFF);
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/983908c8/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
index 5aa11de..f958050 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
@@ -28,7 +28,7 @@ final class OfflinePointWriter implements PointWriter {
final Directory tempDir;
final IndexOutput out;
final int packedBytesLength;
- private long count;
+ long count;
private boolean closed;
// true if ords are written as long (8 bytes), else 4 bytes
private boolean longOrds;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/983908c8/lucene/core/src/java/org/apache/lucene/util/bkd/PointReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/PointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/PointReader.java
index fe7a961..1919f58 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/PointReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/PointReader.java
@@ -20,21 +20,48 @@ package org.apache.lucene.util.bkd;
import java.io.Closeable;
import java.io.IOException;
+import org.apache.lucene.util.LongBitSet;
+
/** One pass iterator through all points previously written with a
* {@link PointWriter}, abstracting away whether points a read
* from (offline) disk or simple arrays in heap. */
-interface PointReader extends Closeable {
+abstract class PointReader implements Closeable {
/** Returns false once iteration is done, else true. */
- boolean next() throws IOException;
+ abstract boolean next() throws IOException;
/** Returns the packed byte[] value */
- byte[] packedValue();
+ abstract byte[] packedValue();
/** Point ordinal */
- long ord();
+ abstract long ord();
/** DocID for this point */
- int docID();
+ abstract int docID();
+
+ /** Splits this reader into left and right partitions */
+ public long split(long count, LongBitSet rightTree, PointWriter left, PointWriter right, boolean doClearBits) throws IOException {
+
+ // Partition this source according to how the splitDim split the values:
+ long rightCount = 0;
+ for (long i=0;i<count;i++) {
+ boolean result = next();
+ assert result;
+ byte[] packedValue = packedValue();
+ long ord = ord();
+ int docID = docID();
+ if (rightTree.get(ord)) {
+ right.append(packedValue, ord, docID);
+ rightCount++;
+ if (doClearBits) {
+ rightTree.clear(ord);
+ }
+ } else {
+ left.append(packedValue, ord, docID);
+ }
+ }
+
+ return rightCount;
+ }
}
[36/50] lucene-solr:jira/SOLR-445: SOLR-8859: read/write Shapes to
String
Posted by ho...@apache.org.
SOLR-8859: read/write Shapes to String
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/022877fe
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/022877fe
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/022877fe
Branch: refs/heads/jira/SOLR-445
Commit: 022877fefabadd5865c335a5b289874d182ed852
Parents: 36145d0
Author: Ryan McKinley <ry...@apache.org>
Authored: Wed Mar 16 12:52:00 2016 -0700
Committer: Ryan McKinley <ry...@apache.org>
Committed: Wed Mar 16 12:52:00 2016 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../solr/schema/AbstractSpatialFieldType.java | 80 +++++++++++++-------
.../org/apache/solr/schema/DateRangeField.java | 4 +-
.../solr/schema/SpatialRPTFieldTypeTest.java | 38 +++++++++-
4 files changed, 95 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/022877fe/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index c48032e..eaedca6 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -45,6 +45,8 @@ New Features
https://github.com/locationtech/spatial4j/blob/master/FORMATS.md
To return the FeatureCollection as the root element, add '&omitHeader=true" (ryan)
+* SOLR-8859: AbstractSpatialFieldType will now convert Shapes to/from Strings
+ using the SpatialContext. (ryan)
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/022877fe/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
index e5fd8c6..7addb20 100644
--- a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
@@ -28,7 +28,6 @@ import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
-
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexableField;
@@ -44,6 +43,7 @@ import org.apache.lucene.spatial.query.SpatialArgsParser;
import org.apache.lucene.spatial.query.SpatialOperation;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
@@ -60,6 +60,9 @@ import com.google.common.cache.CacheBuilder;
import org.locationtech.spatial4j.context.SpatialContext;
import org.locationtech.spatial4j.context.SpatialContextFactory;
import org.locationtech.spatial4j.distance.DistanceUtils;
+import org.locationtech.spatial4j.io.ShapeReader;
+import org.locationtech.spatial4j.io.ShapeWriter;
+import org.locationtech.spatial4j.io.SupportedFormats;
import org.locationtech.spatial4j.shape.Point;
import org.locationtech.spatial4j.shape.Rectangle;
import org.locationtech.spatial4j.shape.Shape;
@@ -83,11 +86,17 @@ public abstract class AbstractSpatialFieldType<T extends SpatialStrategy> extend
public static final String RECIP_DISTANCE = "recipDistance";
public static final String NONE = "none";
+ /** Optional param to pick the string conversion */
+ public static final String FORMAT = "format";
+
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
protected SpatialContext ctx;
protected SpatialArgsParser argsParser;
+ protected ShapeWriter shapeWriter;
+ protected ShapeReader shapeReader;
+
private final Cache<String, T> fieldStrategyCache = CacheBuilder.newBuilder().build();
protected DistanceUnits distanceUnits;
@@ -130,6 +139,25 @@ public abstract class AbstractSpatialFieldType<T extends SpatialStrategy> extend
" on field types with class "+getClass().getSimpleName());
}
+ final SupportedFormats fmts = ctx.getFormats();
+ final String format = args.remove(FORMAT);
+ if (format != null) {
+ shapeWriter = fmts.getWriter(format);
+ shapeReader = fmts.getReader(format);
+ if(shapeWriter==null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Unknown Shape Format: "+ format);
+ }
+ if(shapeReader==null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Unknown Shape Format: "+ format);
+ }
+ }
+ else {
+ // Otherwise, pick the first supported reader/writer
+ shapeWriter = fmts.getWriters().get(0);
+ shapeReader = fmts.getReaders().get(0);
+ }
argsParser = newSpatialArgsParser();
}
@@ -203,38 +231,38 @@ public abstract class AbstractSpatialFieldType<T extends SpatialStrategy> extend
return (shapeStr == null) ? shapeToString(shape) : shapeStr;
}
- protected Shape parseShape(String str) {
+ /** Create a {@link Shape} from the input string */
+ public Shape parseShape(String str) {
if (str.length() == 0)
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "empty string shape");
- if (Character.isLetter(str.charAt(0))) {//WKT starts with a letter
- try {
- return ctx.readShapeFromWkt(str);
- } catch (Exception e) {
- String message = e.getMessage();
- if (!message.contains(str))
- message = "Couldn't parse shape '" + str + "' because: " + message;
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message, e);
- }
- } else {
- return SpatialUtils.parsePointSolrException(str, ctx);
+
+ Shape shape = null;
+ if(shapeReader!=null) {
+ shape = shapeReader.readIfSupported(str);
+ }
+
+ if(shape==null) {
+ // Try all supported formats
+ shape = ctx.getFormats().read(str);
+ }
+
+ if(shape==null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unable to parse shape from: "+str);
}
+ return shape;
}
/**
- * Returns a String version of a shape to be used for the stored value. This method in Solr is only called if for some
- * reason a Shape object is passed to the field type (perhaps via a custom UpdateRequestProcessor),
- * *and* the field is marked as stored. <em>The default implementation throws an exception.</em>
- * <p>
- * Spatial4j 0.4 is probably the last release to support SpatialContext.toString(shape) but it's deprecated with no
- * planned replacement. Shapes do have a toString() method but they are generally internal/diagnostic and not
- * standard WKT.
- * The solution is subclassing and calling ctx.toString(shape) or directly using LegacyShapeReadWriterFormat or
- * passing in some sort of custom wrapped shape that holds a reference to a String or can generate it.
+ * Returns a String version of a shape to be used for the stored value.
+ *
+ * The format can be selected using the initParam <code>format={WKT|GeoJSON}</code>
*/
- protected String shapeToString(Shape shape) {
-// return ctx.toString(shape);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Getting a String from a Shape is no longer possible. See javadocs for commentary.");
+ public String shapeToString(Shape shape) {
+ if(shapeWriter!=null) {
+ return shapeWriter.toString(shape);
+ }
+ // This will only happen if the context does not have any writers
+ throw new SolrException(ErrorCode.SERVER_ERROR, "ShapeWriter not configured");
}
/** Called from {@link #getStrategy(String)} upon first use by fieldName. } */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/022877fe/solr/core/src/java/org/apache/solr/schema/DateRangeField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/DateRangeField.java b/solr/core/src/java/org/apache/solr/schema/DateRangeField.java
index 95b441a..faf049b 100644
--- a/solr/core/src/java/org/apache/solr/schema/DateRangeField.java
+++ b/solr/core/src/java/org/apache/solr/schema/DateRangeField.java
@@ -82,7 +82,7 @@ public class DateRangeField extends AbstractSpatialPrefixTreeFieldType<NumberRan
}
@Override
- protected NRShape parseShape(String str) {
+ public NRShape parseShape(String str) {
if (str.contains(" TO ")) {
//TODO parsing range syntax doesn't support DateMath on either side or exclusive/inclusive
try {
@@ -121,7 +121,7 @@ public class DateRangeField extends AbstractSpatialPrefixTreeFieldType<NumberRan
}
@Override
- protected String shapeToString(Shape shape) {
+ public String shapeToString(Shape shape) {
if (shape instanceof UnitNRShape) {
UnitNRShape unitShape = (UnitNRShape) shape;
if (unitShape.getLevel() == tree.getMaxLevels()) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/022877fe/solr/core/src/test/org/apache/solr/schema/SpatialRPTFieldTypeTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/SpatialRPTFieldTypeTest.java b/solr/core/src/test/org/apache/solr/schema/SpatialRPTFieldTypeTest.java
index 479a7fe..f341832 100644
--- a/solr/core/src/test/org/apache/solr/schema/SpatialRPTFieldTypeTest.java
+++ b/solr/core/src/test/org/apache/solr/schema/SpatialRPTFieldTypeTest.java
@@ -24,6 +24,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.solr.core.AbstractBadConfigTestBase;
import org.junit.After;
import org.junit.Before;
+import org.locationtech.spatial4j.shape.Shape;
public class SpatialRPTFieldTypeTest extends AbstractBadConfigTestBase {
@@ -201,7 +202,35 @@ public class SpatialRPTFieldTypeTest extends AbstractBadConfigTestBase {
);
}
- private void setupRPTField(String distanceUnits, String geo) throws Exception {
+ public void testShapeToFromStringWKT() throws Exception {
+ // Check WKT
+ setupRPTField("miles", "true", "WKT");
+
+ AbstractSpatialFieldType ftype = (AbstractSpatialFieldType)
+ h.getCore().getLatestSchema().getField("geo").getType();
+
+ String wkt = "POINT (1 2)";
+ Shape shape = ftype.parseShape(wkt);
+ String out = ftype.shapeToString(shape);
+
+ assertEquals(wkt, out);
+ }
+
+ public void testShapeToFromStringGeoJSON() throws Exception {
+ // Check WKT
+ setupRPTField("miles", "true", "GeoJSON");
+
+ AbstractSpatialFieldType ftype = (AbstractSpatialFieldType)
+ h.getCore().getLatestSchema().getField("geo").getType();
+
+ String json = "{\"type\":\"Point\",\"coordinates\":[1,2]}";
+ Shape shape = ftype.parseShape(json);
+ String out = ftype.shapeToString(shape);
+
+ assertEquals(json, out);
+ }
+
+ private void setupRPTField(String distanceUnits, String geo, String format) throws Exception {
deleteCore();
File managedSchemaFile = new File(tmpConfDir, "managed-schema");
Files.delete(managedSchemaFile.toPath()); // Delete managed-schema so it won't block parsing a new schema
@@ -220,6 +249,9 @@ public class SpatialRPTFieldTypeTest extends AbstractBadConfigTestBase {
rptMap.put("distanceUnits", distanceUnits);
if(geo!=null)
rptMap.put("geo", geo);
+ if(format!=null) {
+ rptMap.put("format", format);
+ }
rptFieldType.init(oldSchema, rptMap);
rptFieldType.setTypeName("location_rpt");
SchemaField newField = new SchemaField("geo", rptFieldType, SchemaField.STORED | SchemaField.INDEXED, null);
@@ -229,4 +261,8 @@ public class SpatialRPTFieldTypeTest extends AbstractBadConfigTestBase {
assertU(delQ("*:*"));
}
+
+ private void setupRPTField(String distanceUnits, String geo) throws Exception {
+ setupRPTField(distanceUnits, geo, null);
+ }
}
[11/50] lucene-solr:jira/SOLR-445: let BKD use 256 MB heap in 2B tests
Posted by ho...@apache.org.
let BKD use 256 MB heap in 2B tests
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d8eac8e3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d8eac8e3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d8eac8e3
Branch: refs/heads/jira/SOLR-445
Commit: d8eac8e38a46e83010899207bc261fd98a951318
Parents: f474f52
Author: Mike McCandless <mi...@apache.org>
Authored: Sun Mar 13 06:53:24 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Sun Mar 13 06:53:24 2016 -0400
----------------------------------------------------------------------
.../test/org/apache/lucene/util/bkd/Test2BBKDPoints.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d8eac8e3/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
index cf18409..eb3aa47 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
@@ -55,7 +55,7 @@ public class Test2BBKDPoints extends LuceneTestCase {
final int numDocs = (Integer.MAX_VALUE / 26) + 100;
- BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, 1024, 128, Long.BYTES, 26L * numDocs);
+ BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, 1024, 256, Long.BYTES, 26L * numDocs);
int counter = 0;
byte[] packedBytes = new byte[Long.BYTES];
for (int docID = 0; docID < numDocs; docID++) {
@@ -88,8 +88,8 @@ public class Test2BBKDPoints extends LuceneTestCase {
final int numDocs = (Integer.MAX_VALUE / 26) + 100;
- BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES, 26L * numDocs);
- long counter = 0;
+ BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, 1024, 256, Long.BYTES, 26L * numDocs);
+ int counter = 0;
byte[] packedBytes = new byte[2*Long.BYTES];
for (int docID = 0; docID < numDocs; docID++) {
for (int j=0;j<26;j++) {
@@ -98,8 +98,8 @@ public class Test2BBKDPoints extends LuceneTestCase {
// then our counter, which will overflow a bit in the end:
NumericUtils.intToSortableBytes(counter, packedBytes, Integer.BYTES);
// then two random ints for the 2nd dimension:
- NumericUtils.intoSortableBytes(random().nextInt(), packedBytes, Long.BYTES);
- NumericUtils.intoSortableBytes(random().nextInt(), packedBytes, Long.BYTES + Integer.BYTES);
+ NumericUtils.intToSortableBytes(random().nextInt(), packedBytes, Long.BYTES);
+ NumericUtils.intToSortableBytes(random().nextInt(), packedBytes, Long.BYTES + Integer.BYTES);
w.add(packedBytes, docID);
counter++;
}
[10/50] lucene-solr:jira/SOLR-445: improve 2B points test;
add new 2B test against BKD directly
Posted by ho...@apache.org.
improve 2B points test; add new 2B test against BKD directly
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f474f523
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f474f523
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f474f523
Branch: refs/heads/jira/SOLR-445
Commit: f474f523dce537fc5b06e626b259ec4c08faa975
Parents: b466cb6
Author: Mike McCandless <mi...@apache.org>
Authored: Sun Mar 13 06:41:19 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Sun Mar 13 06:41:19 2016 -0400
----------------------------------------------------------------------
.../org/apache/lucene/index/Test2BPoints.java | 9 +-
.../apache/lucene/util/bkd/Test2BBKDPoints.java | 121 +++++++++++++++++++
2 files changed, 127 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f474f523/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
index 43207b8..75f2bbe 100644
--- a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
+++ b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
@@ -70,11 +70,12 @@ public class Test2BPoints extends LuceneTestCase {
}
final int numDocs = (Integer.MAX_VALUE / 26) + 1;
- long counter = 0;
+ int counter = 0;
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
for (int j=0;j<26;j++) {
- doc.add(new LongPoint("long", counter));
+ long x = (((long) random().nextInt() << 32)) | (long) counter;
+ doc.add(new LongPoint("long", x));
counter++;
}
w.addDocument(doc);
@@ -120,7 +121,9 @@ public class Test2BPoints extends LuceneTestCase {
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
for (int j=0;j<26;j++) {
- doc.add(new LongPoint("long", counter, 2*counter+1));
+ long x = (((long) random().nextInt() << 32)) | (long) counter;
+ long y = (((long) random().nextInt() << 32)) | (long) random().nextInt();
+ doc.add(new LongPoint("long", x, y));
counter++;
}
w.addDocument(doc);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f474f523/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
new file mode 100644
index 0000000..cf18409
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util.bkd;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PointsReader;
+import org.apache.lucene.codecs.PointsWriter;
+import org.apache.lucene.codecs.lucene60.Lucene60PointsReader;
+import org.apache.lucene.codecs.lucene60.Lucene60PointsWriter;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.LuceneTestCase.Monster;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.TimeUnits;
+
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
+// e.g. run like this: ant test -Dtestcase=Test2BBKDPoints -Dtests.nightly=true -Dtests.verbose=true -Dtests.monster=true
+//
+// or: python -u /l/util/src/python/repeatLuceneTest.py -heap 4g -once -nolog -tmpDir /b/tmp -logDir /l/logs Test2BBKDPoints.test2D -verbose
+
+@TimeoutSuite(millis = 365 * 24 * TimeUnits.HOUR) // hopefully ~1 year is long enough ;)
+@Monster("takes at least 4 hours and consumes many GB of temp disk space")
+public class Test2BBKDPoints extends LuceneTestCase {
+ public void test1D() throws Exception {
+ Directory dir = FSDirectory.open(createTempDir("2BBKDPoints1D"));
+
+ final int numDocs = (Integer.MAX_VALUE / 26) + 100;
+
+ BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, 1024, 128, Long.BYTES, 26L * numDocs);
+ int counter = 0;
+ byte[] packedBytes = new byte[Long.BYTES];
+ for (int docID = 0; docID < numDocs; docID++) {
+ for (int j=0;j<26;j++) {
+ // first a random int:
+ NumericUtils.intToSortableBytes(random().nextInt(), packedBytes, 0);
+ // then our counter, which will overflow a bit in the end:
+ NumericUtils.intToSortableBytes(counter, packedBytes, Integer.BYTES);
+ w.add(packedBytes, docID);
+ counter++;
+ }
+ if (VERBOSE && docID % 100000 == 0) {
+ System.out.println(docID + " of " + numDocs + "...");
+ }
+ }
+ IndexOutput out = dir.createOutput("1d.bkd", IOContext.DEFAULT);
+ long indexFP = w.finish(out);
+ out.close();
+
+ IndexInput in = dir.openInput("1d.bkd", IOContext.DEFAULT);
+ in.seek(indexFP);
+ BKDReader r = new BKDReader(in);
+ r.verify(numDocs);
+ in.close();
+ dir.close();
+ }
+
+ public void test2D() throws Exception {
+ Directory dir = FSDirectory.open(createTempDir("2BBKDPoints2D"));
+
+ final int numDocs = (Integer.MAX_VALUE / 26) + 100;
+
+ BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES, 26L * numDocs);
+ long counter = 0;
+ byte[] packedBytes = new byte[2*Long.BYTES];
+ for (int docID = 0; docID < numDocs; docID++) {
+ for (int j=0;j<26;j++) {
+ // first a random int:
+ NumericUtils.intToSortableBytes(random().nextInt(), packedBytes, 0);
+ // then our counter, which will overflow a bit in the end:
+ NumericUtils.intToSortableBytes(counter, packedBytes, Integer.BYTES);
+ // then two random ints for the 2nd dimension:
+ NumericUtils.intoSortableBytes(random().nextInt(), packedBytes, Long.BYTES);
+ NumericUtils.intoSortableBytes(random().nextInt(), packedBytes, Long.BYTES + Integer.BYTES);
+ w.add(packedBytes, docID);
+ counter++;
+ }
+ if (VERBOSE && docID % 100000 == 0) {
+ System.out.println(docID + " of " + numDocs + "...");
+ }
+ }
+ IndexOutput out = dir.createOutput("2d.bkd", IOContext.DEFAULT);
+ long indexFP = w.finish(out);
+ out.close();
+
+ IndexInput in = dir.openInput("2d.bkd", IOContext.DEFAULT);
+ in.seek(indexFP);
+ BKDReader r = new BKDReader(in);
+ r.verify(numDocs);
+ in.close();
+ dir.close();
+ }
+}
[07/50] lucene-solr:jira/SOLR-445: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Posted by ho...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fa970073
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fa970073
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fa970073
Branch: refs/heads/jira/SOLR-445
Commit: fa9700737a5351219731030eeb65b37d5dccb962
Parents: b420ad4 41ef29a
Author: Mike McCandless <mi...@apache.org>
Authored: Sun Mar 13 05:35:34 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Sun Mar 13 05:35:34 2016 -0400
----------------------------------------------------------------------
.../simpletext/SimpleTextPointsWriter.java | 5 +-
.../org/apache/lucene/util/bkd/BKDWriter.java | 87 ++++++++++----------
.../apache/lucene/util/bkd/HeapPointReader.java | 1 -
.../apache/lucene/util/bkd/HeapPointWriter.java | 10 +++
4 files changed, 57 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
[09/50] lucene-solr:jira/SOLR-445: make BKD's temp file names a bit
more descriptive
Posted by ho...@apache.org.
make BKD's temp file names a bit more descriptive
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b466cb63
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b466cb63
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b466cb63
Branch: refs/heads/jira/SOLR-445
Commit: b466cb637627bab15276facd32c7398cad573e47
Parents: fcd90b9
Author: Mike McCandless <mi...@apache.org>
Authored: Sun Mar 13 06:28:49 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Sun Mar 13 06:28:49 2016 -0400
----------------------------------------------------------------------
.../src/java/org/apache/lucene/util/bkd/BKDWriter.java | 10 +++++-----
.../org/apache/lucene/util/bkd/OfflinePointWriter.java | 4 ++--
2 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b466cb63/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 33d7bc4..d4e30b7 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -216,7 +216,7 @@ public class BKDWriter implements Closeable {
private void switchToOffline() throws IOException {
// For each .add we just append to this input file, then in .finish we sort this input and resursively build the tree:
- offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds);
+ offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds, "switch");
tempInput = offlinePointWriter.out;
PointReader reader = heapPointWriter.getReader(0);
for(int i=0;i<pointCount;i++) {
@@ -1172,8 +1172,8 @@ public class BKDWriter implements Closeable {
continue;
}
- try (PointWriter leftPointWriter = getPointWriter(leftCount);
- PointWriter rightPointWriter = getPointWriter(source.count - leftCount);
+ try (PointWriter leftPointWriter = getPointWriter(leftCount, "left" + dim);
+ PointWriter rightPointWriter = getPointWriter(source.count - leftCount, "right" + dim);
PointReader reader = slices[dim].writer.getReader(slices[dim].start);) {
// Partition this source according to how the splitDim split the values:
@@ -1238,12 +1238,12 @@ public class BKDWriter implements Closeable {
return true;
}
- PointWriter getPointWriter(long count) throws IOException {
+ PointWriter getPointWriter(long count, String desc) throws IOException {
if (count <= maxPointsSortInHeap) {
int size = Math.toIntExact(count);
return new HeapPointWriter(size, size, packedBytesLength, longOrds);
} else {
- return new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds);
+ return new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds, desc);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b466cb63/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
index dcf6781..5aa11de 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
@@ -33,8 +33,8 @@ final class OfflinePointWriter implements PointWriter {
// true if ords are written as long (8 bytes), else 4 bytes
private boolean longOrds;
- public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int packedBytesLength, boolean longOrds) throws IOException {
- this.out = tempDir.createTempOutput(tempFileNamePrefix, "bkd", IOContext.DEFAULT);
+ public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int packedBytesLength, boolean longOrds, String desc) throws IOException {
+ this.out = tempDir.createTempOutput(tempFileNamePrefix, "bkd_" + desc, IOContext.DEFAULT);
this.tempDir = tempDir;
this.packedBytesLength = packedBytesLength;
this.longOrds = longOrds;
[05/50] lucene-solr:jira/SOLR-445: optimize BKD leaf block writing:
use incoming sorted points to compute commonn prefix (saves one pass);
remove an extra copy bytes
Posted by ho...@apache.org.
optimize BKD leaf block writing: use incoming sorted points to compute commonn prefix (saves one pass); remove an extra copy bytes
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/41ef29a2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/41ef29a2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/41ef29a2
Branch: refs/heads/jira/SOLR-445
Commit: 41ef29a2c39241113cb999d9c4b2fbb3e70a40af
Parents: 576a405
Author: Mike McCandless <mi...@apache.org>
Authored: Sun Mar 13 05:31:11 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Sun Mar 13 05:31:11 2016 -0400
----------------------------------------------------------------------
.../simpletext/SimpleTextPointsWriter.java | 5 +-
.../org/apache/lucene/util/bkd/BKDWriter.java | 87 ++++++++++----------
.../apache/lucene/util/bkd/HeapPointReader.java | 1 -
.../apache/lucene/util/bkd/HeapPointWriter.java | 10 +++
4 files changed, 57 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/41ef29a2/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java
index 13494f5..33af33e 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java
@@ -158,11 +158,10 @@ class SimpleTextPointsWriter extends PointsWriter {
}
@Override
- protected void writeLeafBlockPackedValue(IndexOutput out, int[] commonPrefixLengths, byte[] bytes) throws IOException {
+ protected void writeLeafBlockPackedValue(IndexOutput out, int[] commonPrefixLengths, byte[] bytes, int bytesOffset) throws IOException {
// NOTE: we don't do prefix coding, so we ignore commonPrefixLengths
- assert bytes.length == packedBytesLength;
write(out, BLOCK_VALUE);
- write(out, new BytesRef(bytes, 0, bytes.length).toString());
+ write(out, new BytesRef(bytes, bytesOffset, packedBytesLength).toString());
newline(out);
}
}) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/41ef29a2/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 6d3cf03..765b01c 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -106,9 +106,9 @@ public class BKDWriter implements Closeable {
final double maxMBSortInHeap;
final byte[] scratchDiff;
- final byte[] scratchPackedValue;
final byte[] scratch1;
final byte[] scratch2;
+ final BytesRef scratchBytesRef = new BytesRef();
final int[] commonPrefixLengths;
protected final FixedBitSet docsSeen;
@@ -152,7 +152,7 @@ public class BKDWriter implements Closeable {
packedBytesLength = numDims * bytesPerDim;
scratchDiff = new byte[bytesPerDim];
- scratchPackedValue = new byte[packedBytesLength];
+ scratchBytesRef.length = packedBytesLength;
scratch1 = new byte[packedBytesLength];
scratch2 = new byte[packedBytesLength];
commonPrefixLengths = new int[numDims];
@@ -455,7 +455,7 @@ public class BKDWriter implements Closeable {
}
System.arraycopy(reader.state.scratchPackedValue, 0, maxPackedValue, 0, packedBytesLength);
- assert numDims > 1 || valueInOrder(valueCount, lastPackedValue, reader.state.scratchPackedValue);
+ assert numDims > 1 || valueInOrder(valueCount, lastPackedValue, reader.state.scratchPackedValue, 0);
valueCount++;
if (pointCount > totalPointCount) {
throw new IllegalStateException("totalPointCount=" + totalPointCount + " was passed when we were created, but we just hit " + pointCount + " values");
@@ -502,7 +502,7 @@ public class BKDWriter implements Closeable {
// Write the full values:
for (int i=0;i<leafCount;i++) {
- writeLeafBlockPackedValue(out, commonPrefixLengths, leafBlockPackedValues[i]);
+ writeLeafBlockPackedValue(out, commonPrefixLengths, leafBlockPackedValues[i], 0);
}
leafCount = 0;
@@ -920,10 +920,10 @@ public class BKDWriter implements Closeable {
}
}
- protected void writeLeafBlockPackedValue(IndexOutput out, int[] commonPrefixLengths, byte[] bytes) throws IOException {
+ protected void writeLeafBlockPackedValue(IndexOutput out, int[] commonPrefixLengths, byte[] bytes, int offset) throws IOException {
for(int dim=0;dim<numDims;dim++) {
int prefix = commonPrefixLengths[dim];
- out.writeBytes(bytes, dim*bytesPerDim+prefix, bytesPerDim-prefix);
+ out.writeBytes(bytes, offset+dim*bytesPerDim+prefix, bytesPerDim-prefix);
}
}
@@ -994,13 +994,13 @@ public class BKDWriter implements Closeable {
}
/** Called only in assert */
- private boolean valueInBounds(byte[] packedValue, byte[] minPackedValue, byte[] maxPackedValue) {
+ private boolean valueInBounds(BytesRef packedValue, byte[] minPackedValue, byte[] maxPackedValue) {
for(int dim=0;dim<numDims;dim++) {
int offset = bytesPerDim*dim;
- if (StringHelper.compare(bytesPerDim, packedValue, offset, minPackedValue, offset) < 0) {
+ if (StringHelper.compare(bytesPerDim, packedValue.bytes, packedValue.offset + offset, minPackedValue, offset) < 0) {
return false;
}
- if (StringHelper.compare(bytesPerDim, packedValue, offset, maxPackedValue, offset) > 0) {
+ if (StringHelper.compare(bytesPerDim, packedValue.bytes, packedValue.offset + offset, maxPackedValue, offset) > 0) {
return false;
}
}
@@ -1060,16 +1060,35 @@ public class BKDWriter implements Closeable {
}
if (nodeID >= leafNodeOffset) {
+
// Leaf node: write block
+ for (int dim=0;dim<numDims;dim++) {
+ if (slices[dim].writer instanceof HeapPointWriter == false) {
+ // Adversarial cases can cause this, e.g. very lopsided data, all equal points, such that we started
+ // offline, but then kept splitting only in one dimension, and so never had to rewrite into heap writer
+ slices[dim] = switchToHeap(slices[dim]);
+ }
- PathSlice source = slices[0];
+ PathSlice source = slices[dim];
+
+ HeapPointWriter heapSource = (HeapPointWriter) source.writer;
- if (source.writer instanceof HeapPointWriter == false) {
- // Adversarial cases can cause this, e.g. very lopsided data, all equal points, such that we started
- // offline, but then kept splitting only in one dimension, and so never had to rewrite into heap writer
- source = switchToHeap(source);
+ // Find common prefix by comparing first and last values, already sorted in this dimension:
+ heapSource.readPackedValue(Math.toIntExact(source.start), scratch1);
+ heapSource.readPackedValue(Math.toIntExact(source.start + source.count - 1), scratch2);
+
+ int offset = dim * bytesPerDim;
+ commonPrefixLengths[dim] = bytesPerDim;
+ for(int j=0;j<bytesPerDim;j++) {
+ if (scratch1[offset+j] != scratch2[offset+j]) {
+ commonPrefixLengths[dim] = j;
+ break;
+ }
+ }
}
+ PathSlice source = slices[0];
+
// We ensured that maxPointsSortInHeap was >= maxPointsInLeafNode, so we better be in heap at this point:
HeapPointWriter heapSource = (HeapPointWriter) source.writer;
@@ -1083,37 +1102,21 @@ public class BKDWriter implements Closeable {
assert count > 0: "nodeID=" + nodeID + " leafNodeOffset=" + leafNodeOffset;
writeLeafBlockDocs(out, heapSource.docIDs, Math.toIntExact(source.start), count);
- // First pass: find the per-dim common prefix for all values in this block:
- Arrays.fill(commonPrefixLengths, bytesPerDim);
- for (int i=0;i<count;i++) {
- if (i == 0) {
- heapSource.readPackedValue(Math.toIntExact(source.start + i), scratch1);
- } else {
- heapSource.readPackedValue(Math.toIntExact(source.start + i), scratchPackedValue);
- for(int dim=0;dim<numDims;dim++) {
- int offset = dim * bytesPerDim;
- for(int j=0;j<commonPrefixLengths[dim];j++) {
- if (scratch1[offset+j] != scratchPackedValue[offset+j]) {
- commonPrefixLengths[dim] = j;
- break;
- }
- }
- }
- }
- }
+ // TODO: minor opto: we don't really have to write the actual common prefixes, because BKDReader on recursing can regenerate it for us
+ // from the index, much like how terms dict does so from the FST:
+ // Write the common prefixes:
writeCommonPrefixes(out, commonPrefixLengths, scratch1);
- // Second pass: write the full values:
+ // Write the full values:
byte[] lastPackedValue = new byte[bytesPerDim];
for (int i=0;i<count;i++) {
- // TODO: we could do bulk copying here, avoiding the intermediate copy:
- heapSource.readPackedValue(Math.toIntExact(source.start + i), scratchPackedValue);
- assert numDims != 1 || valueInOrder(i, lastPackedValue, scratchPackedValue);
+ heapSource.getPackedValueSlice(Math.toIntExact(source.start + i), scratchBytesRef);
+ assert numDims != 1 || valueInOrder(i, lastPackedValue, scratchBytesRef.bytes, scratchBytesRef.offset);
// Make sure this value does in fact fall within this leaf cell:
- assert valueInBounds(scratchPackedValue, minPackedValue, maxPackedValue);
- writeLeafBlockPackedValue(out, commonPrefixLengths, scratchPackedValue);
+ assert valueInBounds(scratchBytesRef, minPackedValue, maxPackedValue);
+ writeLeafBlockPackedValue(out, commonPrefixLengths, scratchBytesRef.bytes, scratchBytesRef.offset);
}
} else {
@@ -1227,11 +1230,11 @@ public class BKDWriter implements Closeable {
}
// only called from assert
- private boolean valueInOrder(long ord, byte[] lastPackedValue, byte[] packedValue) {
- if (ord > 0 && StringHelper.compare(bytesPerDim, lastPackedValue, 0, packedValue, 0) > 0) {
- throw new AssertionError("values out of order: last value=" + new BytesRef(lastPackedValue) + " current value=" + new BytesRef(packedValue) + " ord=" + ord);
+ private boolean valueInOrder(long ord, byte[] lastPackedValue, byte[] packedValue, int packedValueOffset) {
+ if (ord > 0 && StringHelper.compare(bytesPerDim, lastPackedValue, 0, packedValue, packedValueOffset) > 0) {
+ throw new AssertionError("values out of order: last value=" + new BytesRef(lastPackedValue) + " current value=" + new BytesRef(packedValue, packedValueOffset, packedBytesLength) + " ord=" + ord);
}
- System.arraycopy(packedValue, 0, lastPackedValue, 0, bytesPerDim);
+ System.arraycopy(packedValue, packedValueOffset, lastPackedValue, 0, bytesPerDim);
return true;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/41ef29a2/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
index b178f08..63c7869 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
@@ -16,7 +16,6 @@
*/
package org.apache.lucene.util.bkd;
-
import java.util.List;
import org.apache.lucene.util.PagedBytes;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/41ef29a2/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
index 3b043d0..45bb591 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
@@ -20,6 +20,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
final class HeapPointWriter implements PointWriter {
int[] docIDs;
@@ -72,6 +73,15 @@ final class HeapPointWriter implements PointWriter {
System.arraycopy(blocks.get(block), blockIndex * packedBytesLength, bytes, 0, packedBytesLength);
}
+ /** Returns a reference, in <code>result</code>, to the byte[] slice holding this value */
+ void getPackedValueSlice(int index, BytesRef result) {
+ int block = index / valuesPerBlock;
+ int blockIndex = index % valuesPerBlock;
+ result.bytes = blocks.get(block);
+ result.offset = blockIndex * packedBytesLength;
+ assert result.length == packedBytesLength;
+ }
+
void writePackedValue(int index, byte[] bytes) {
assert bytes.length == packedBytesLength;
int block = index / valuesPerBlock;
[34/50] lucene-solr:jira/SOLR-445: LUCENE-7106: Add helpers to
compute aggregated stats on points.
Posted by ho...@apache.org.
LUCENE-7106: Add helpers to compute aggregated stats on points.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/24830b7f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/24830b7f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/24830b7f
Branch: refs/heads/jira/SOLR-445
Commit: 24830b7f18146b38078a80bc04f041011ab8689e
Parents: 2c8b2a6
Author: Adrien Grand <jp...@gmail.com>
Authored: Wed Mar 16 15:34:40 2016 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Wed Mar 16 15:34:40 2016 +0100
----------------------------------------------------------------------
.../org/apache/lucene/index/PointValues.java | 90 +++++++++++++++++++-
.../apache/lucene/index/TestPointValues.java | 62 ++++++++++++--
2 files changed, 145 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/24830b7f/lucene/core/src/java/org/apache/lucene/index/PointValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValues.java b/lucene/core/src/java/org/apache/lucene/index/PointValues.java
index 1fb2654..a4fd323 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PointValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PointValues.java
@@ -23,10 +23,10 @@ import java.net.InetAddress;
import org.apache.lucene.document.BinaryPoint;
import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.bkd.BKDWriter;
/**
@@ -86,6 +86,94 @@ public abstract class PointValues {
/** Maximum number of dimensions */
public static final int MAX_DIMENSIONS = BKDWriter.MAX_DIMS;
+ /** Return the cumulated number of points across all leaves of the given
+ * {@link IndexReader}.
+ * @see PointValues#size(String) */
+ public static long size(IndexReader reader, String field) throws IOException {
+ long size = 0;
+ for (LeafReaderContext ctx : reader.leaves()) {
+ PointValues values = ctx.reader().getPointValues();
+ if (values != null) {
+ size += values.size(field);
+ }
+ }
+ return size;
+ }
+
+ /** Return the cumulated number of docs that have points across all leaves
+ * of the given {@link IndexReader}.
+ * @see PointValues#getDocCount(String) */
+ public static int getDocCount(IndexReader reader, String field) throws IOException {
+ int count = 0;
+ for (LeafReaderContext ctx : reader.leaves()) {
+ PointValues values = ctx.reader().getPointValues();
+ if (values != null) {
+ count += values.getDocCount(field);
+ }
+ }
+ return count;
+ }
+
+ /** Return the minimum packed values across all leaves of the given
+ * {@link IndexReader}.
+ * @see PointValues#getMinPackedValue(String) */
+ public static byte[] getMinPackedValue(IndexReader reader, String field) throws IOException {
+ byte[] minValue = null;
+ for (LeafReaderContext ctx : reader.leaves()) {
+ PointValues values = ctx.reader().getPointValues();
+ if (values == null) {
+ continue;
+ }
+ byte[] leafMinValue = values.getMinPackedValue(field);
+ if (leafMinValue == null) {
+ continue;
+ }
+ if (minValue == null) {
+ minValue = leafMinValue.clone();
+ } else {
+ final int numDimensions = values.getNumDimensions(field);
+ final int numBytesPerDimension = values.getBytesPerDimension(field);
+ for (int i = 0; i < numDimensions; ++i) {
+ int offset = i * numBytesPerDimension;
+ if (StringHelper.compare(numBytesPerDimension, leafMinValue, offset, minValue, offset) < 0) {
+ System.arraycopy(leafMinValue, offset, minValue, offset, numBytesPerDimension);
+ }
+ }
+ }
+ }
+ return minValue;
+ }
+
+ /** Return the maximum packed values across all leaves of the given
+ * {@link IndexReader}.
+ * @see PointValues#getMaxPackedValue(String) */
+ public static byte[] getMaxPackedValue(IndexReader reader, String field) throws IOException {
+ byte[] maxValue = null;
+ for (LeafReaderContext ctx : reader.leaves()) {
+ PointValues values = ctx.reader().getPointValues();
+ if (values == null) {
+ continue;
+ }
+ byte[] leafMaxValue = values.getMaxPackedValue(field);
+ if (leafMaxValue == null) {
+ continue;
+ }
+ if (maxValue == null) {
+ maxValue = leafMaxValue.clone();
+ } else {
+ final int numDimensions = values.getNumDimensions(field);
+ final int numBytesPerDimension = values.getBytesPerDimension(field);
+ for (int i = 0; i < numDimensions; ++i) {
+ int offset = i * numBytesPerDimension;
+ if (StringHelper.compare(numBytesPerDimension, leafMaxValue, offset, maxValue, offset) > 0) {
+ System.arraycopy(leafMaxValue, offset, maxValue, offset, numBytesPerDimension);
+ }
+ }
+ }
+ }
+ return maxValue;
+ }
+
/** Default constructor */
protected PointValues() {
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/24830b7f/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
index 49cbc2a..c7ca2dc 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
@@ -22,12 +22,6 @@ import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.PointsFormat;
-import org.apache.lucene.codecs.PointsReader;
-import org.apache.lucene.codecs.PointsWriter;
-import org.apache.lucene.codecs.lucene60.Lucene60PointsReader;
-import org.apache.lucene.codecs.lucene60.Lucene60PointsWriter;
import org.apache.lucene.document.BinaryPoint;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoublePoint;
@@ -657,4 +651,60 @@ public class TestPointValues extends LuceneTestCase {
assertTrue(output.toString(IOUtils.UTF_8).contains("test: points..."));
dir.close();
}
+
+ public void testMergedStats() throws IOException {
+ final int iters = atLeast(3);
+ for (int iter = 0; iter < iters; ++iter) {
+ doTestMergedStats();
+ }
+ }
+
+ private static byte[][] randomBinaryValue(int numDims, int numBytesPerDim) {
+ byte[][] bytes = new byte[numDims][];
+ for (int i = 0; i < numDims; ++i) {
+ bytes[i] = new byte[numBytesPerDim];
+ random().nextBytes(bytes[i]);
+ }
+ return bytes;
+ }
+
+ private void doTestMergedStats() throws IOException {
+ final int numDims = TestUtil.nextInt(random(), 1, 8);
+ final int numBytesPerDim = TestUtil.nextInt(random(), 1, 16);
+ Directory dir = new RAMDirectory();
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
+ final int numDocs = TestUtil.nextInt(random(), 10, 20);
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = new Document();
+ final int numPoints = random().nextInt(3);
+ for (int j = 0; j < numPoints; ++j) {
+ doc.add(new BinaryPoint("field", randomBinaryValue(numDims, numBytesPerDim)));
+ }
+ w.addDocument(doc);
+ if (random().nextBoolean()) {
+ DirectoryReader.open(w).close();
+ }
+ }
+
+ final IndexReader reader1 = DirectoryReader.open(w);
+ w.forceMerge(1);
+ final IndexReader reader2 = DirectoryReader.open(w);
+ final PointValues expected = getOnlyLeafReader(reader2).getPointValues();
+ if (expected == null) {
+ assertNull(PointValues.getMinPackedValue(reader1, "field"));
+ assertNull(PointValues.getMaxPackedValue(reader1, "field"));
+ assertEquals(0, PointValues.getDocCount(reader1, "field"));
+ assertEquals(0, PointValues.size(reader1, "field"));
+ } else {
+ assertArrayEquals(
+ expected.getMinPackedValue("field"),
+ PointValues.getMinPackedValue(reader1, "field"));
+ assertArrayEquals(
+ expected.getMaxPackedValue("field"),
+ PointValues.getMaxPackedValue(reader1, "field"));
+ assertEquals(expected.getDocCount("field"), PointValues.getDocCount(reader1, "field"));
+ assertEquals(expected.size("field"), PointValues.size(reader1, "field"));
+ }
+ IOUtils.close(w, reader1, reader2, dir);
+ }
}
[49/50] lucene-solr:jira/SOLR-445: SOLR-445: cleanup some simple
nocommits
Posted by ho...@apache.org.
SOLR-445: cleanup some simple nocommits
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6ec8c635
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6ec8c635
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6ec8c635
Branch: refs/heads/jira/SOLR-445
Commit: 6ec8c635bf5853dfb229f89cb2818749c1cfe8ce
Parents: 1aa1ba3
Author: Chris Hostetter <ho...@apache.org>
Authored: Sun Mar 20 16:05:52 2016 -0700
Committer: Chris Hostetter <ho...@apache.org>
Committed: Sun Mar 20 16:05:52 2016 -0700
----------------------------------------------------------------------
.../processor/DistributedUpdateProcessor.java | 24 +--
.../processor/TolerantUpdateProcessor.java | 29 ++--
.../TolerantUpdateProcessorFactory.java | 20 ++-
.../conf/solrconfig-tolerant-update-minimal.xml | 40 +++++
.../org/apache/solr/core/TestBadConfig.java | 5 +
.../processor/TolerantUpdateProcessorTest.java | 14 +-
.../solr/common/ToleratedUpdateError.java | 24 ++-
.../solr/common/TestToleratedUpdateError.java | 160 ++++++++++++++-----
8 files changed, 248 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ec8c635/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
index 01aa38b..0c7836e 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
@@ -1722,19 +1722,23 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
private static final int buildCode(List<Error> errors) {
assert null != errors;
assert 0 < errors.size();
-
- // if they are all the same, then we use that...
- int result = errors.get(0).statusCode;
+
+ int minCode = Integer.MAX_VALUE;
+ int maxCode = Integer.MIN_VALUE;
for (Error error : errors) {
log.trace("REMOTE ERROR: {}", error);
- if (result != error.statusCode ) {
- // ...otherwise use sensible default
- return ErrorCode.SERVER_ERROR.code;
- // nocommit: don't short circut - check them all...
- // nocommit: ...even if not all same, use 400 if all 4xx, else use 500
- }
+ minCode = Math.min(error.statusCode, minCode);
+ maxCode = Math.max(error.statusCode, maxCode);
}
- return result;
+ if (minCode == maxCode) {
+ // all codes are consistent, use that...
+ return minCode;
+ } else if (400 <= minCode && maxCode < 500) {
+ // all codes are 4xx, use 400
+ return ErrorCode.BAD_REQUEST.code;
+ }
+ // ...otherwise use sensible default
+ return ErrorCode.SERVER_ERROR.code;
}
/** Helper method for constructor */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ec8c635/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java
index e53d33c..78457bb 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java
@@ -77,10 +77,11 @@ import org.slf4j.LoggerFactory;
*/
public class TolerantUpdateProcessor extends UpdateRequestProcessor {
private static final Logger log = LoggerFactory.getLogger(TolerantUpdateProcessor.class);
+
/**
- * String to be used as document key in the response if a real ID can't be determined
+ * String to be used as document key for errors when a real uniqueKey can't be determined
*/
- private static final String UNKNOWN_ID = "(unknown)"; // nocommit: fail hard and fast if no uniqueKey
+ private static final String UNKNOWN_ID = "(unknown)";
/**
* Response Header
@@ -93,7 +94,10 @@ public class TolerantUpdateProcessor extends UpdateRequestProcessor {
* batch
*/
private final int maxErrors;
-
+
+ /** The uniqueKey field */
+ private SchemaField uniqueKeyField;
+
private final SolrQueryRequest req;
private ZkController zkController;
@@ -137,8 +141,8 @@ public class TolerantUpdateProcessor extends UpdateRequestProcessor {
assert ! DistribPhase.FROMLEADER.equals(distribPhase);
this.zkController = this.req.getCore().getCoreDescriptor().getCoreContainer().getZkController();
-
- // nocommit: assert existence of uniqueKey field & record for future use
+ this.uniqueKeyField = this.req.getCore().getLatestSchema().getUniqueKeyField();
+ assert null != uniqueKeyField : "Factory didn't enforce uniqueKey field?";
}
@Override
@@ -160,7 +164,7 @@ public class TolerantUpdateProcessor extends UpdateRequestProcessor {
knownErrors.add(new ToleratedUpdateError
(CmdType.ADD,
- getPrintableId(id, cmd.getReq().getSchema().getUniqueKeyField()),
+ getPrintableId(id),
t.getMessage()));
if (knownErrors.size() > maxErrors) {
firstErrTracker.throwFirst();
@@ -319,15 +323,14 @@ public class TolerantUpdateProcessor extends UpdateRequestProcessor {
* Returns the output of {@link org.apache.solr.schema.FieldType#
* indexedToReadable(BytesRef, CharsRefBuilder)} of the field
* type of the uniqueKey on the {@link BytesRef} passed as parameter.
- * <code>ref</code> should be the indexed representation of the id and
- * <code>field</code> should be the uniqueKey schema field. If any of
- * the two parameters is null this method will return {@link #UNKNOWN_ID}
+ * <code>ref</code> should be the indexed representation of the id -- if null
+ * (possibly because it's missing in the update) this method will return {@link #UNKNOWN_ID}
*/
- private String getPrintableId(BytesRef ref, SchemaField field) {
- if(ref == null || field == null) {
- return UNKNOWN_ID; // nocommit: fail hard and fast
+ private String getPrintableId(BytesRef ref) {
+ if (ref == null) {
+ return UNKNOWN_ID;
}
- return field.getType().indexedToReadable(ref, new CharsRefBuilder()).toString();
+ return uniqueKeyField.getType().indexedToReadable(ref, new CharsRefBuilder()).toString();
}
// nocommit: 1) is this method even needed? 2) is this method correct? 3) javadocs
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ec8c635/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessorFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessorFactory.java
index d049077..35ca63b 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessorFactory.java
@@ -19,9 +19,12 @@ package org.apache.solr.update.processor;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.SchemaField;
import org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase;
+import org.apache.solr.util.plugin.SolrCoreAware;
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
@@ -68,9 +71,7 @@ import static org.apache.solr.update.processor.DistributingUpdateProcessorFactor
*
*/
public class TolerantUpdateProcessorFactory extends UpdateRequestProcessorFactory
- implements UpdateRequestProcessorFactory.RunAlways {
-
- // nocommit: make SolrCoreAware and fail fast if no uniqueKey configured
+ implements SolrCoreAware, UpdateRequestProcessorFactory.RunAlways {
/**
* Parameter that defines how many errors the UpdateRequestProcessor will tolerate
@@ -82,6 +83,8 @@ public class TolerantUpdateProcessorFactory extends UpdateRequestProcessorFactor
* or in the request
*/
private int defaultMaxErrors = Integer.MAX_VALUE;
+
+ private boolean informed = false;
@SuppressWarnings("rawtypes")
@Override
@@ -101,8 +104,19 @@ public class TolerantUpdateProcessorFactory extends UpdateRequestProcessorFactor
}
@Override
+ public void inform(SolrCore core) {
+ informed = true;
+ if (null == core.getLatestSchema().getUniqueKeyField()) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, this.getClass().getName() +
+ " requires a schema that includes a uniqueKey field.");
+ }
+ }
+
+ @Override
public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ assert informed : "inform(SolrCore) never called?";
+
// short circut if we're a replica processing commands from our leader
DistribPhase distribPhase = DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
if (DistribPhase.FROMLEADER.equals(distribPhase)) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ec8c635/solr/core/src/test-files/solr/collection1/conf/solrconfig-tolerant-update-minimal.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tolerant-update-minimal.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tolerant-update-minimal.xml
new file mode 100644
index 0000000..d3b90db
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tolerant-update-minimal.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<config>
+
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+ <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="indent">true</str>
+ <str name="df">text</str>
+ </lst>
+ </requestHandler>
+
+ <updateRequestProcessorChain name="tolerant-chain">
+ <processor class="solr.TolerantUpdateProcessorFactory" />
+ <processor class="solr.RunUpdateProcessorFactory" />
+ </updateRequestProcessorChain>
+
+</config>
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ec8c635/solr/core/src/test/org/apache/solr/core/TestBadConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestBadConfig.java b/solr/core/src/test/org/apache/solr/core/TestBadConfig.java
index 637c975..31361be 100644
--- a/solr/core/src/test/org/apache/solr/core/TestBadConfig.java
+++ b/solr/core/src/test/org/apache/solr/core/TestBadConfig.java
@@ -96,4 +96,9 @@ public class TestBadConfig extends AbstractBadConfigTestBase {
assertConfigs("bad-solrconfig-unexpected-schema-attribute.xml", "schema-minimal.xml",
"Unexpected arg(s): {bogusParam=bogusValue}");
}
+
+ public void testTolerantUpdateProcessorNoUniqueKey() throws Exception {
+ assertConfigs("solrconfig-tolerant-update-minimal.xml", "schema-minimal.xml",
+ "requires a schema that includes a uniqueKey field");
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ec8c635/solr/core/src/test/org/apache/solr/update/processor/TolerantUpdateProcessorTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/TolerantUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/TolerantUpdateProcessorTest.java
index a519068..9bbead8 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/TolerantUpdateProcessorTest.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/TolerantUpdateProcessorTest.java
@@ -242,7 +242,19 @@ public class TolerantUpdateProcessorTest extends UpdateProcessorTestBase {
,"//result[@numFound='6']");
}
- // nocommit: need a testMaxErrorsNegative (ie: infinite)
+ @Test
+ public void testMaxErrorsInfinite() throws IOException {
+ ModifiableSolrParams requestParams = new ModifiableSolrParams();
+ requestParams.add("maxErrors", "-1");
+ try {
+ assertAddsSucceedWithErrors("tolerant-chain-max-errors-not-set", docs, null, badIds);
+ } catch(Exception e) {
+ fail("Shouldn't get an exception for this batch: " + e.getMessage());
+ }
+ assertU(commit());
+ assertQ(req("q","*:*")
+ ,"//result[@numFound='10']");
+ }
@Test
public void testMaxErrors0() throws IOException {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ec8c635/solr/solrj/src/java/org/apache/solr/common/ToleratedUpdateError.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/ToleratedUpdateError.java b/solr/solrj/src/java/org/apache/solr/common/ToleratedUpdateError.java
index c6c4244..6da6fc5 100644
--- a/solr/solrj/src/java/org/apache/solr/common/ToleratedUpdateError.java
+++ b/solr/solrj/src/java/org/apache/solr/common/ToleratedUpdateError.java
@@ -19,10 +19,13 @@ package org.apache.solr.common;
import java.util.ArrayList;
import java.util.List;
import org.apache.solr.common.util.SimpleOrderedMap;
-
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
/**
- * nocommit: more javadocs, mention (but obviously no link) to TolerantUpdateProcessor
+ * Models the basic information related to a single "tolerated" error that occured during updates.
+ * This class is only useful when the <code>ToleranteUpdateProcessorFactory</code> is used in an update
+ * processor chain
*/
public final class ToleratedUpdateError {
@@ -71,8 +74,17 @@ public final class ToleratedUpdateError {
* @see #getSimpleMap
*/
public static ToleratedUpdateError parseMap(SimpleOrderedMap<String> data) {
- // nocommit: error handling and clean exception reporting if data is bogus
- return new ToleratedUpdateError(CmdType.valueOf(data.get("type")), data.get("id"), data.get("message"));
+ final String id = data.get("id");
+ final String message = data.get("message");
+ final String t = data.get("type");
+ if (null == t || null == id || null == message) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Map does not represent a ToleratedUpdateError, must contain 'type', 'id', and 'message'");
+ }
+ try {
+ return new ToleratedUpdateError(CmdType.valueOf(t), id, message);
+ } catch (IllegalArgumentException iae) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid type for ToleratedUpdateError: " + t, iae);
+ }
}
/**
@@ -86,7 +98,9 @@ public final class ToleratedUpdateError {
return null; // not a key we care about
}
final int typeEnd = metadataKey.indexOf(':', META_PRE_LEN);
- assert 0 < typeEnd; // nocommit: better error handling
+ if (typeEnd < 0) {
+ return null; // has our prefix, but not our format -- must not be a key we (actually) care about
+ }
return new ToleratedUpdateError(CmdType.valueOf(metadataKey.substring(META_PRE_LEN, typeEnd)),
metadataKey.substring(typeEnd+1), metadataVal);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ec8c635/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java b/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java
index 91636b3..a76443a 100644
--- a/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java
+++ b/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java
@@ -16,67 +16,120 @@
*/
package org.apache.solr.common;
+import java.util.EnumSet;
import org.apache.solr.common.ToleratedUpdateError;
import org.apache.solr.common.ToleratedUpdateError.CmdType;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
/** Basic testing of the serialization/encapsulation code in ToleratedUpdateError */
public class TestToleratedUpdateError extends LuceneTestCase {
- // nocommit: add randomized testing, particularly with non-trivial 'id' values
+ private final static CmdType[] ALL_TYPES = EnumSet.allOf(CmdType.class).toArray(new CmdType[0]);
+
+ public void testBasics() {
+
+ assertFalse((new ToleratedUpdateError(CmdType.ADD, "doc1", "some error")).equals
+ (new ToleratedUpdateError(CmdType.ADD, "doc2", "some error")));
+ assertFalse((new ToleratedUpdateError(CmdType.ADD, "doc1", "some error")).equals
+ (new ToleratedUpdateError(CmdType.ADD, "doc1", "some errorxx")));
+ assertFalse((new ToleratedUpdateError(CmdType.ADD, "doc1", "some error")).equals
+ (new ToleratedUpdateError(CmdType.DELID, "doc1", "some error")));
+ }
- public void checkRoundTripComparisons(Coppier coppier) {
+ public void testParseMetadataErrorHandling() {
assertNull(ToleratedUpdateError.parseMetadataIfToleratedUpdateError("some other key", "some value"));
+
+ // see if someone tries to trick us into having an NPE...
+ ToleratedUpdateError valid = new ToleratedUpdateError(CmdType.ADD, "doc2", "some error");
+ String badKey = valid.getMetadataKey().replace(":", "X");
+ assertNull(ToleratedUpdateError.parseMetadataIfToleratedUpdateError(badKey, valid.getMetadataValue()));
+ }
+
+ public void testParseMapErrorChecking() {
+ SimpleOrderedMap<String> bogus = new SimpleOrderedMap<String>();
+ try {
+ ToleratedUpdateError.parseMap(bogus);
+ fail("map should not be parsable");
+ } catch (SolrException e) {
+ assertTrue(e.toString(), e.getMessage().contains("Map does not represent a ToleratedUpdateError") );
+ }
+
+ bogus.add("id", "some id");
+ bogus.add("message", "some message");
+ try {
+ ToleratedUpdateError.parseMap(bogus);
+ fail("map should still not be parsable");
+ } catch (SolrException e) {
+ assertTrue(e.toString(), e.getMessage().contains("Map does not represent a ToleratedUpdateError") );
+ }
+ bogus.add("type", "not a real type");
+ try {
+ ToleratedUpdateError.parseMap(bogus);
+ fail("invalid type should not be parsable");
+ } catch (SolrException e) {
+ assertTrue(e.toString(), e.getMessage().contains("Invalid type"));
+ }
+ }
+
+ public void testParseMap() {
+ // trivial
+ SimpleOrderedMap valid = new SimpleOrderedMap<String>();
+ valid.add("type", CmdType.ADD.toString());
+ valid.add("id", "some id");
+ valid.add("message", "some message");
+
+ ToleratedUpdateError in = ToleratedUpdateError.parseMap(valid);
+ compare(in, MAP_COPPIER);
+ compare(in, METADATA_COPPIER);
+
+ // randomized
+ int numIters = atLeast(5000);
+ for (int i = 0; i < numIters; i++) {
+ valid = new SimpleOrderedMap<String>();
+ valid.add("type", ALL_TYPES[TestUtil.nextInt(random(), 0, ALL_TYPES.length-1)].toString());
+ valid.add("id", TestUtil.randomUnicodeString(random()));
+ valid.add("message", TestUtil.randomUnicodeString(random()));
+
+ in = ToleratedUpdateError.parseMap(valid);
+ compare(in, MAP_COPPIER);
+ compare(in, METADATA_COPPIER);
+ }
+ }
+
+ public void checkRoundTripComparisons(Coppier coppier) {
+
+ // some simple basics
for (ToleratedUpdateError in : new ToleratedUpdateError[] {
new ToleratedUpdateError(CmdType.ADD, "doc1", "some error"),
new ToleratedUpdateError(CmdType.DELID, "doc1", "some diff error"),
new ToleratedUpdateError(CmdType.DELQ, "-field:yakko other_field:wakko", "some other error"),
}) {
- ToleratedUpdateError out = coppier.copy(in);
-
- assertNotNull(out);
- assertEquals(out.type, in.type);
- assertEquals(out.id, in.id);
- assertEquals(out.errorValue, in.errorValue);
- assertEquals(out.hashCode(), in.hashCode());
- assertEquals(out.toString(), in.toString());
-
- assertEquals(in.getMetadataKey(), out.getMetadataKey());
- assertEquals(in.getMetadataValue(), out.getMetadataValue());
-
- assertEquals(out, in);
- assertEquals(in, out);
+ compare(in, coppier);
+ }
+ // randomized testing of non trivial keys/values
+ int numIters = atLeast(5000);
+ for (int i = 0; i < numIters; i++) {
+ ToleratedUpdateError in = new ToleratedUpdateError
+ (ALL_TYPES[TestUtil.nextInt(random(), 0, ALL_TYPES.length-1)],
+ TestUtil.randomUnicodeString(random()),
+ TestUtil.randomUnicodeString(random()));
+ compare(in, coppier);
}
-
- assertFalse((new ToleratedUpdateError(CmdType.ADD, "doc1", "some error")).equals
- (new ToleratedUpdateError(CmdType.ADD, "doc2", "some error")));
- assertFalse((new ToleratedUpdateError(CmdType.ADD, "doc1", "some error")).equals
- (new ToleratedUpdateError(CmdType.ADD, "doc1", "some errorxx")));
- assertFalse((new ToleratedUpdateError(CmdType.ADD, "doc1", "some error")).equals
- (new ToleratedUpdateError(CmdType.DELID, "doc1", "some error")));
-
}
-
+
public void testMetadataRoundTripComparisons(Coppier coppier) {
- checkRoundTripComparisons(new Coppier() {
- public ToleratedUpdateError copy(ToleratedUpdateError in) {
- return ToleratedUpdateError.parseMetadataIfToleratedUpdateError
- (in.getMetadataKey(), in.getMetadataValue());
- }
- });
+ checkRoundTripComparisons(METADATA_COPPIER);
}
public void testMapRoundTripComparisons() {
- checkRoundTripComparisons(new Coppier() {
- public ToleratedUpdateError copy(ToleratedUpdateError in) {
- return ToleratedUpdateError.parseMap(in.getSimpleMap());
- }
- });
+ checkRoundTripComparisons(MAP_COPPIER);
}
/** trivial sanity check */
@@ -95,9 +148,44 @@ public class TestToleratedUpdateError extends LuceneTestCase {
}
+ public void compare(ToleratedUpdateError in, Coppier coppier) {
+ ToleratedUpdateError out = coppier.copy(in);
+ assertNotNull(out);
+ compare(in, out);
+ }
+
+ public void compare(ToleratedUpdateError in, ToleratedUpdateError out) {
+ assertEquals(out.type, in.type);
+ assertEquals(out.id, in.id);
+ assertEquals(out.errorValue, in.errorValue);
+
+ assertEquals(out.hashCode(), in.hashCode());
+ assertEquals(out.toString(), in.toString());
+
+ assertEquals(in.getMetadataKey(), out.getMetadataKey());
+ assertEquals(in.getMetadataValue(), out.getMetadataValue());
+
+ assertEquals(out, in);
+ assertEquals(in, out);
+ }
+
private static abstract class Coppier {
public abstract ToleratedUpdateError copy(ToleratedUpdateError in);
}
+
+ private static final Coppier MAP_COPPIER = new Coppier() {
+ public ToleratedUpdateError copy(ToleratedUpdateError in) {
+ return ToleratedUpdateError.parseMap(in.getSimpleMap());
+ }
+ };
+
+ private static final Coppier METADATA_COPPIER = new Coppier() {
+ public ToleratedUpdateError copy(ToleratedUpdateError in) {
+ return ToleratedUpdateError.parseMetadataIfToleratedUpdateError
+ (in.getMetadataKey(), in.getMetadataValue());
+ }
+ };
+
}
[20/50] lucene-solr:jira/SOLR-445: LUCENE-7103: further optimize
LatLonPoint.newDistanceSort
Posted by ho...@apache.org.
LUCENE-7103: further optimize LatLonPoint.newDistanceSort
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1660b563
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1660b563
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1660b563
Branch: refs/heads/jira/SOLR-445
Commit: 1660b5630aec516fb7365e8cb5c0a8a2bfd14d2f
Parents: 0f949c8
Author: Robert Muir <rm...@apache.org>
Authored: Mon Mar 14 16:25:31 2016 -0400
Committer: Robert Muir <rm...@apache.org>
Committed: Mon Mar 14 16:25:31 2016 -0400
----------------------------------------------------------------------
.../document/LatLonPointDistanceComparator.java | 41 ++++++++++++++++----
.../lucene/document/TestBigIntegerPoint.java | 4 +-
.../lucene/document/TestInetAddressPoint.java | 4 +-
.../apache/lucene/document/TestLatLonPoint.java | 2 +-
.../document/TestLatLonPointDistanceQuery.java | 4 +-
.../document/TestLatLonPointDistanceSort.java | 28 +++++++++++--
6 files changed, 64 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1660b563/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
index 86c9134..ef4c3f3 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
@@ -26,9 +26,10 @@ import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.Scorer;
-import org.apache.lucene.spatial.util.GeoDistanceUtils;
+import org.apache.lucene.spatial.util.GeoProjectionUtils;
import org.apache.lucene.spatial.util.GeoRect;
import org.apache.lucene.spatial.util.GeoUtils;
+import org.apache.lucene.util.SloppyMath;
/**
* Compares documents by distance from an origin point
@@ -96,7 +97,7 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
crossesDateLine = false;
} else {
assert Double.isFinite(bottom);
- GeoRect box = GeoUtils.circleToBBox(longitude, latitude, bottom);
+ GeoRect box = GeoUtils.circleToBBox(longitude, latitude, haversin2(bottom));
// pre-encode our box to our integer encoding, so we don't have to decode
// to double values for uncompetitive hits. This has some cost!
int minLatEncoded = LatLonPoint.encodeLatitude(box.minLat);
@@ -166,7 +167,7 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
if (outsideBox == false) {
double docLatitude = LatLonPoint.decodeLatitude(latitudeBits);
double docLongitude = LatLonPoint.decodeLongitude(longitudeBits);
- minValue = Math.min(minValue, GeoDistanceUtils.haversin(latitude, longitude, docLatitude, docLongitude));
+ minValue = Math.min(minValue, haversin1(latitude, longitude, docLatitude, docLongitude));
}
}
return Double.compare(bottom, minValue);
@@ -174,7 +175,7 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
@Override
public void copy(int slot, int doc) throws IOException {
- values[slot] = distance(doc);
+ values[slot] = sortKey(doc);
}
@Override
@@ -190,17 +191,17 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
@Override
public Double value(int slot) {
- return Double.valueOf(values[slot]);
+ return Double.valueOf(haversin2(values[slot]));
}
@Override
public int compareTop(int doc) throws IOException {
- return Double.compare(topValue, distance(doc));
+ return Double.compare(topValue, haversin2(sortKey(doc)));
}
// TODO: optimize for single-valued case?
// TODO: do all kinds of other optimizations!
- double distance(int doc) {
+ double sortKey(int doc) {
currentDocs.setDocument(doc);
int numValues = currentDocs.count();
@@ -213,8 +214,32 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
long encoded = currentDocs.valueAt(i);
double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32));
double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF));
- minValue = Math.min(minValue, GeoDistanceUtils.haversin(latitude, longitude, docLatitude, docLongitude));
+ minValue = Math.min(minValue, haversin1(latitude, longitude, docLatitude, docLongitude));
}
return minValue;
}
+
+ // sort by first part of the haversin computation. note that this value is meaningless to the user.
+ // invoke haversin2() to "complete" the calculation and get a distance in meters.
+ static double haversin1(double lat1, double lon1, double lat2, double lon2) {
+ double dLat = SloppyMath.TO_RADIANS * (lat2 - lat1);
+ double dLon = SloppyMath.TO_RADIANS * (lon2 - lon1);
+ lat1 = SloppyMath.TO_RADIANS * (lat1);
+ lat2 = SloppyMath.TO_RADIANS * (lat2);
+
+ final double sinDLatO2 = SloppyMath.sin(dLat / 2);
+ final double sinDLonO2 = SloppyMath.sin(dLon / 2);
+
+ return sinDLatO2*sinDLatO2 + sinDLonO2 * sinDLonO2 * SloppyMath.cos(lat1) * SloppyMath.cos(lat2);
+ }
+
+ // second half of the haversin calculation, used to convert results from haversin1 (used internally
+ // for sorting) for display purposes.
+ static double haversin2(double partial) {
+ if (Double.isInfinite(partial)) {
+ return partial;
+ }
+ double c = 2 * SloppyMath.asin(Math.sqrt(partial));
+ return (GeoProjectionUtils.SEMIMAJOR_AXIS * c);
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1660b563/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java
index 8f38bcd..a7a1295 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java
@@ -41,7 +41,7 @@ public class TestBigIntegerPoint extends LuceneTestCase {
// search and verify we found our doc
IndexReader reader = writer.getReader();
- IndexSearcher searcher = newSearcher(reader, false);
+ IndexSearcher searcher = newSearcher(reader);
assertEquals(1, searcher.count(BigIntegerPoint.newExactQuery("field", large)));
assertEquals(1, searcher.count(BigIntegerPoint.newRangeQuery("field", large.subtract(BigInteger.ONE), large.add(BigInteger.ONE))));
assertEquals(1, searcher.count(BigIntegerPoint.newSetQuery("field", large)));
@@ -66,7 +66,7 @@ public class TestBigIntegerPoint extends LuceneTestCase {
// search and verify we found our doc
IndexReader reader = writer.getReader();
- IndexSearcher searcher = newSearcher(reader, false);
+ IndexSearcher searcher = newSearcher(reader);
assertEquals(1, searcher.count(BigIntegerPoint.newExactQuery("field", negative)));
assertEquals(1, searcher.count(BigIntegerPoint.newRangeQuery("field", negative.subtract(BigInteger.ONE), negative.add(BigInteger.ONE))));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1660b563/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java
index c91b52b..b0e7107 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java
@@ -41,7 +41,7 @@ public class TestInetAddressPoint extends LuceneTestCase {
// search and verify we found our doc
IndexReader reader = writer.getReader();
- IndexSearcher searcher = newSearcher(reader, false);
+ IndexSearcher searcher = newSearcher(reader);
assertEquals(1, searcher.count(InetAddressPoint.newExactQuery("field", address)));
assertEquals(1, searcher.count(InetAddressPoint.newPrefixQuery("field", address, 24)));
assertEquals(1, searcher.count(InetAddressPoint.newRangeQuery("field", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5"))));
@@ -68,7 +68,7 @@ public class TestInetAddressPoint extends LuceneTestCase {
// search and verify we found our doc
IndexReader reader = writer.getReader();
- IndexSearcher searcher = newSearcher(reader, false);
+ IndexSearcher searcher = newSearcher(reader);
assertEquals(1, searcher.count(InetAddressPoint.newExactQuery("field", address)));
assertEquals(1, searcher.count(InetAddressPoint.newPrefixQuery("field", address, 64)));
assertEquals(1, searcher.count(InetAddressPoint.newRangeQuery("field", InetAddress.getByName("fec0::f66c"), InetAddress.getByName("fec0::f66e"))));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1660b563/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java
index d180d58..ff4af12 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java
@@ -39,7 +39,7 @@ public class TestLatLonPoint extends LuceneTestCase {
// search and verify we found our doc
IndexReader reader = writer.getReader();
- IndexSearcher searcher = newSearcher(reader, false);
+ IndexSearcher searcher = newSearcher(reader);
assertEquals(1, searcher.count(LatLonPoint.newBoxQuery("field", 18, 19, -66, -65)));
reader.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1660b563/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java
index fa95710..c69791c 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java
@@ -55,7 +55,7 @@ public class TestLatLonPointDistanceQuery extends LuceneTestCase {
// search within 50km and verify we found our doc
IndexReader reader = writer.getReader();
- IndexSearcher searcher = newSearcher(reader, false);
+ IndexSearcher searcher = newSearcher(reader);
assertEquals(1, searcher.count(LatLonPoint.newDistanceQuery("field", 18, -65, 50_000)));
reader.close();
@@ -148,7 +148,7 @@ public class TestLatLonPointDistanceQuery extends LuceneTestCase {
writer.addDocument(doc);
}
IndexReader reader = writer.getReader();
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
for (int i = 0; i < numQueries; i++) {
double lat = -90 + 180.0 * random().nextDouble();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1660b563/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
index a776b3f..7df956f 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
@@ -54,7 +54,7 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
iw.addDocument(doc);
IndexReader reader = iw.getReader();
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
iw.close();
Sort sort = new Sort(LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731));
@@ -91,7 +91,7 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
iw.addDocument(doc);
IndexReader reader = iw.getReader();
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
iw.close();
Sort sort = new Sort(LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731));
@@ -128,7 +128,7 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
iw.addDocument(doc);
IndexReader reader = iw.getReader();
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
iw.close();
SortField sortField = LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731);
@@ -234,7 +234,7 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
writer.addDocument(doc);
}
IndexReader reader = writer.getReader();
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
for (int i = 0; i < numQueries; i++) {
double lat = -90 + 180.0 * random().nextDouble();
@@ -289,4 +289,24 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
writer.close();
dir.close();
}
+
+ /** Test this method sorts the same way as real haversin */
+ public void testPartialHaversin() {
+ for (int i = 0; i < 100000; i++) {
+ double centerLat = -90 + 180.0 * random().nextDouble();
+ double centerLon = -180 + 360.0 * random().nextDouble();
+
+ double lat1 = -90 + 180.0 * random().nextDouble();
+ double lon1 = -180 + 360.0 * random().nextDouble();
+
+ double lat2 = -90 + 180.0 * random().nextDouble();
+ double lon2 = -180 + 360.0 * random().nextDouble();
+
+ int expected = Integer.signum(Double.compare(GeoDistanceUtils.haversin(centerLat, centerLon, lat1, lon1),
+ GeoDistanceUtils.haversin(centerLat, centerLon, lat2, lon2)));
+ int actual = Integer.signum(Double.compare(LatLonPointDistanceComparator.haversin1(centerLat, centerLon, lat1, lon1),
+ LatLonPointDistanceComparator.haversin1(centerLat, centerLon, lat2, lon2)));
+ assertEquals(expected, actual);
+ }
+ }
}
[42/50] lucene-solr:jira/SOLR-445: add asserts
Posted by ho...@apache.org.
add asserts
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/85945ef2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/85945ef2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/85945ef2
Branch: refs/heads/jira/SOLR-445
Commit: 85945ef2a78bf8dc7fa037ad5f41455f462c6396
Parents: 6ebf615
Author: Mike McCandless <mi...@apache.org>
Authored: Thu Mar 17 10:10:28 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Thu Mar 17 10:10:28 2016 -0400
----------------------------------------------------------------------
lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85945ef2/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 10d97e3..fd3408d 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -977,12 +977,14 @@ public class BKDWriter implements Closeable {
System.arraycopy(reader.packedValue(), splitDim*bytesPerDim, scratch1, 0, bytesPerDim);
if (numDims > 1) {
+ assert ordBitSet.get(reader.ord()) == false;
ordBitSet.set(reader.ord());
// Start at 1 because we already did the first value above (so we could keep the split value):
for(int i=1;i<rightCount;i++) {
result = reader.next();
assert result;
+ assert ordBitSet.get(reader.ord()) == false;
ordBitSet.set(reader.ord());
}
}
[03/50] lucene-solr:jira/SOLR-445: remove O(N^2) asserts
Posted by ho...@apache.org.
remove O(N^2) asserts
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0ff341f7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0ff341f7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0ff341f7
Branch: refs/heads/jira/SOLR-445
Commit: 0ff341f747f9ff035a305272a07bd123ca890a0d
Parents: b8cfcaf
Author: Mike McCandless <mi...@apache.org>
Authored: Sat Mar 12 06:17:03 2016 -0500
Committer: Mike McCandless <mi...@apache.org>
Committed: Sat Mar 12 06:17:03 2016 -0500
----------------------------------------------------------------------
lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ff341f7/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index bb2402b..6d3cf03 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -970,7 +970,6 @@ public class BKDWriter implements Closeable {
private byte[] markRightTree(long rightCount, int splitDim, PathSlice source, LongBitSet ordBitSet) throws IOException {
// Now we mark ords that fall into the right half, so we can partition on all other dims that are not the split dim:
- assert numDims == 1 || ordBitSet.cardinality() == 0: "cardinality=" + ordBitSet.cardinality();
// Read the split value, then mark all ords in the right tree (larger than the split value):
try (PointReader reader = source.writer.getReader(source.start + source.count - rightCount)) {
@@ -988,8 +987,6 @@ public class BKDWriter implements Closeable {
assert result;
ordBitSet.set(reader.ord());
}
-
- assert rightCount == ordBitSet.cardinality(): "rightCount=" + rightCount + " cardinality=" + ordBitSet.cardinality();
}
}
[33/50] lucene-solr:jira/SOLR-445: remove obselete warning: this sort
makes bounding boxes for you
Posted by ho...@apache.org.
remove obselete warning: this sort makes bounding boxes for you
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2c8b2a6c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2c8b2a6c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2c8b2a6c
Branch: refs/heads/jira/SOLR-445
Commit: 2c8b2a6cd3e02053bac2ca1a8f761079435eaf08
Parents: 6ea458a
Author: Robert Muir <rm...@apache.org>
Authored: Wed Mar 16 09:30:45 2016 -0400
Committer: Robert Muir <rm...@apache.org>
Committed: Wed Mar 16 09:30:45 2016 -0400
----------------------------------------------------------------------
.../sandbox/src/java/org/apache/lucene/document/LatLonPoint.java | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2c8b2a6c/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
index f5541bd..7c056e2 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
@@ -336,10 +336,6 @@ public class LatLonPoint extends Field {
* (missing values sort last).
* <p>
* If a document contains multiple values for the field, the <i>closest</i> distance to the location is used.
- * <p>
- * <b>NOTE</b>: distance sorting might be expensive for many documents. Consider restricting the document
- * set with a {@link #newBoxQuery box}, {@link #newDistanceQuery radius} radius, or {@link #newPolygonQuery polygon}
- * query for better performance
*
* @param field field name. cannot be null.
* @param latitude latitude at the center: must be within standard +/-90 coordinate bounds.
[41/50] lucene-solr:jira/SOLR-445: SOLR-7339: Upgrade to Jetty
9.3.8.v20160314
Posted by ho...@apache.org.
SOLR-7339: Upgrade to Jetty 9.3.8.v20160314
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6ebf6153
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6ebf6153
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6ebf6153
Branch: refs/heads/jira/SOLR-445
Commit: 6ebf61535e90d264755ba72eea9ce51ea89703ff
Parents: ae846bf
Author: Steve Rowe <sa...@apache.org>
Authored: Thu Mar 17 01:46:36 2016 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Thu Mar 17 01:46:36 2016 -0400
----------------------------------------------------------------------
lucene/ivy-versions.properties | 2 +-
lucene/licenses/jetty-continuation-9.3.6.v20151106.jar.sha1 | 1 -
lucene/licenses/jetty-continuation-9.3.8.v20160314.jar.sha1 | 1 +
lucene/licenses/jetty-http-9.3.6.v20151106.jar.sha1 | 1 -
lucene/licenses/jetty-http-9.3.8.v20160314.jar.sha1 | 1 +
lucene/licenses/jetty-io-9.3.6.v20151106.jar.sha1 | 1 -
lucene/licenses/jetty-io-9.3.8.v20160314.jar.sha1 | 1 +
lucene/licenses/jetty-server-9.3.6.v20151106.jar.sha1 | 1 -
lucene/licenses/jetty-server-9.3.8.v20160314.jar.sha1 | 1 +
lucene/licenses/jetty-servlet-9.3.6.v20151106.jar.sha1 | 1 -
lucene/licenses/jetty-servlet-9.3.8.v20160314.jar.sha1 | 1 +
lucene/licenses/jetty-util-9.3.6.v20151106.jar.sha1 | 1 -
lucene/licenses/jetty-util-9.3.8.v20160314.jar.sha1 | 1 +
solr/CHANGES.txt | 4 ++--
solr/licenses/jetty-continuation-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-continuation-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/jetty-deploy-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-deploy-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/jetty-http-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-http-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/jetty-io-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-io-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/jetty-jmx-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-jmx-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/jetty-rewrite-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-rewrite-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/jetty-security-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-security-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/jetty-server-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-server-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/jetty-servlet-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-servlet-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/jetty-servlets-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-servlets-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/jetty-util-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-util-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/jetty-webapp-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-webapp-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/jetty-xml-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-xml-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/start.jar.sha1 | 2 +-
41 files changed, 23 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index d5ef256..c228577 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -228,7 +228,7 @@ org.codehaus.jackson.version = 1.9.13
/org.codehaus.woodstox/woodstox-core-asl = 4.4.1
/org.easymock/easymock = 3.0
-org.eclipse.jetty.version = 9.3.6.v20151106
+org.eclipse.jetty.version = 9.3.8.v20160314
/org.eclipse.jetty/jetty-continuation = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-deploy = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-http = ${org.eclipse.jetty.version}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/lucene/licenses/jetty-continuation-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/jetty-continuation-9.3.6.v20151106.jar.sha1 b/lucene/licenses/jetty-continuation-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index 6e16eda..0000000
--- a/lucene/licenses/jetty-continuation-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a120bc737d2efc6ebf4a703325ee679aff181881
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/lucene/licenses/jetty-continuation-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/jetty-continuation-9.3.8.v20160314.jar.sha1 b/lucene/licenses/jetty-continuation-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..9197ba8
--- /dev/null
+++ b/lucene/licenses/jetty-continuation-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+dec4dfc43617637694762822ef99c8373c944c98
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/lucene/licenses/jetty-http-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/jetty-http-9.3.6.v20151106.jar.sha1 b/lucene/licenses/jetty-http-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index cd8fdfc..0000000
--- a/lucene/licenses/jetty-http-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c2bba60bc1f9fe5779ac20ab30232bf9a89d3e52
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/lucene/licenses/jetty-http-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/jetty-http-9.3.8.v20160314.jar.sha1 b/lucene/licenses/jetty-http-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..c36a298
--- /dev/null
+++ b/lucene/licenses/jetty-http-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+0127feb7407f4137ff4295b5fa2895845db56710
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/lucene/licenses/jetty-io-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/jetty-io-9.3.6.v20151106.jar.sha1 b/lucene/licenses/jetty-io-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index b06088c..0000000
--- a/lucene/licenses/jetty-io-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-09e59bde867e55d8c93cdd682d12317733ef5339
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/lucene/licenses/jetty-io-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/jetty-io-9.3.8.v20160314.jar.sha1 b/lucene/licenses/jetty-io-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..b49fa7a
--- /dev/null
+++ b/lucene/licenses/jetty-io-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+371e3c2b72d9a9737579ec0fdfd6a2a3ab8b8141
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/lucene/licenses/jetty-server-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/jetty-server-9.3.6.v20151106.jar.sha1 b/lucene/licenses/jetty-server-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index e154577..0000000
--- a/lucene/licenses/jetty-server-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d9c43a1b20ede7e3c456237d71b4cce1dff5457a
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/lucene/licenses/jetty-server-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/jetty-server-9.3.8.v20160314.jar.sha1 b/lucene/licenses/jetty-server-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..0def5cb
--- /dev/null
+++ b/lucene/licenses/jetty-server-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+da8366f602f35d4c3177cb081472e2fc4abe04ea
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/lucene/licenses/jetty-servlet-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/jetty-servlet-9.3.6.v20151106.jar.sha1 b/lucene/licenses/jetty-servlet-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index f28a4c2..0000000
--- a/lucene/licenses/jetty-servlet-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-62c03d6c7203735d4e28e4e78e22df38152f01ef
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/lucene/licenses/jetty-servlet-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/jetty-servlet-9.3.8.v20160314.jar.sha1 b/lucene/licenses/jetty-servlet-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..7180159
--- /dev/null
+++ b/lucene/licenses/jetty-servlet-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+ea5f25d3326d7745d9c21d405dcf6f878efbd5fb
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/lucene/licenses/jetty-util-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/jetty-util-9.3.6.v20151106.jar.sha1 b/lucene/licenses/jetty-util-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index 48b6b12..0000000
--- a/lucene/licenses/jetty-util-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8721c8e670c11ea19005c567733453956b6243fc
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/lucene/licenses/jetty-util-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/jetty-util-9.3.8.v20160314.jar.sha1 b/lucene/licenses/jetty-util-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..026b5d0
--- /dev/null
+++ b/lucene/licenses/jetty-util-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+01d53c7a7e7715e67d6f4edec6c5b328ee162e65
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index b5aa670..ef5d422 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -74,7 +74,7 @@ Carrot2 3.12.0
Velocity 1.7 and Velocity Tools 2.0
Apache UIMA 2.3.1
Apache ZooKeeper 3.4.6
-Jetty 9.3.6.v20151106
+Jetty 9.3.8.v20160314
System Requirements
----------------------
@@ -409,7 +409,7 @@ Other Changes
* SOLR-8529: Improve JdbcTest to not use plain assert statements (Kevin Risden, Joel Bernstein)
-* SOLR-7339: Upgrade Jetty to v9.3.6.v20151106. (Gregg Donovan, shalin, Mark Miller)
+* SOLR-7339: Upgrade Jetty to v9.3.8.v20160314. (Gregg Donovan, shalin, Mark Miller, Steve Rowe)
* SOLR-5730: Make Lucene's SortingMergePolicy and EarlyTerminatingSortingCollector configurable in Solr.
(Christine Poerschke, hossmann, Tomás Fernández Löbbe, Shai Erera)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-continuation-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-continuation-9.3.6.v20151106.jar.sha1 b/solr/licenses/jetty-continuation-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index 6e16eda..0000000
--- a/solr/licenses/jetty-continuation-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a120bc737d2efc6ebf4a703325ee679aff181881
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-continuation-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-continuation-9.3.8.v20160314.jar.sha1 b/solr/licenses/jetty-continuation-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..9197ba8
--- /dev/null
+++ b/solr/licenses/jetty-continuation-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+dec4dfc43617637694762822ef99c8373c944c98
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-deploy-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-deploy-9.3.6.v20151106.jar.sha1 b/solr/licenses/jetty-deploy-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index e65d127..0000000
--- a/solr/licenses/jetty-deploy-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8a4813aacd2dda3aa36b109d7fe338abdd413239
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-deploy-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-deploy-9.3.8.v20160314.jar.sha1 b/solr/licenses/jetty-deploy-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..dea43a0
--- /dev/null
+++ b/solr/licenses/jetty-deploy-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+fe4025121641f5c4b06986e9b14983964bfcd7d5
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-http-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-http-9.3.6.v20151106.jar.sha1 b/solr/licenses/jetty-http-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index cd8fdfc..0000000
--- a/solr/licenses/jetty-http-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c2bba60bc1f9fe5779ac20ab30232bf9a89d3e52
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-http-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-http-9.3.8.v20160314.jar.sha1 b/solr/licenses/jetty-http-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..c36a298
--- /dev/null
+++ b/solr/licenses/jetty-http-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+0127feb7407f4137ff4295b5fa2895845db56710
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-io-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-io-9.3.6.v20151106.jar.sha1 b/solr/licenses/jetty-io-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index b06088c..0000000
--- a/solr/licenses/jetty-io-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-09e59bde867e55d8c93cdd682d12317733ef5339
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-io-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-io-9.3.8.v20160314.jar.sha1 b/solr/licenses/jetty-io-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..b49fa7a
--- /dev/null
+++ b/solr/licenses/jetty-io-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+371e3c2b72d9a9737579ec0fdfd6a2a3ab8b8141
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-jmx-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-jmx-9.3.6.v20151106.jar.sha1 b/solr/licenses/jetty-jmx-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index f36781b..0000000
--- a/solr/licenses/jetty-jmx-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c96ec3bbee1e3ff277929e3aff2126de5b9748c1
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-jmx-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-jmx-9.3.8.v20160314.jar.sha1 b/solr/licenses/jetty-jmx-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..0da16de
--- /dev/null
+++ b/solr/licenses/jetty-jmx-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+4aca2eb607d49969bac6a5f36be24ebe1d6d39ad
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-rewrite-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-rewrite-9.3.6.v20151106.jar.sha1 b/solr/licenses/jetty-rewrite-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index 7a5b5a7..0000000
--- a/solr/licenses/jetty-rewrite-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-75f233e85377fa476f210423014bc8c20824e4c5
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-rewrite-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-rewrite-9.3.8.v20160314.jar.sha1 b/solr/licenses/jetty-rewrite-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..670f11ea2
--- /dev/null
+++ b/solr/licenses/jetty-rewrite-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+264a34089a62d22cea8e38f6ab6c55d8cef992dc
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-security-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-security-9.3.6.v20151106.jar.sha1 b/solr/licenses/jetty-security-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index e7ad9b2..0000000
--- a/solr/licenses/jetty-security-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e44ffc80834a7f78a5b0ed15c54b875956772242
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-security-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-security-9.3.8.v20160314.jar.sha1 b/solr/licenses/jetty-security-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..a209f1f
--- /dev/null
+++ b/solr/licenses/jetty-security-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+5291fa5e3098f08017bfcc7f950a7ce36c9544d7
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-server-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-server-9.3.6.v20151106.jar.sha1 b/solr/licenses/jetty-server-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index e154577..0000000
--- a/solr/licenses/jetty-server-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d9c43a1b20ede7e3c456237d71b4cce1dff5457a
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-server-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-server-9.3.8.v20160314.jar.sha1 b/solr/licenses/jetty-server-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..0def5cb
--- /dev/null
+++ b/solr/licenses/jetty-server-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+da8366f602f35d4c3177cb081472e2fc4abe04ea
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-servlet-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-servlet-9.3.6.v20151106.jar.sha1 b/solr/licenses/jetty-servlet-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index f28a4c2..0000000
--- a/solr/licenses/jetty-servlet-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-62c03d6c7203735d4e28e4e78e22df38152f01ef
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-servlet-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-servlet-9.3.8.v20160314.jar.sha1 b/solr/licenses/jetty-servlet-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..7180159
--- /dev/null
+++ b/solr/licenses/jetty-servlet-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+ea5f25d3326d7745d9c21d405dcf6f878efbd5fb
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-servlets-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-servlets-9.3.6.v20151106.jar.sha1 b/solr/licenses/jetty-servlets-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index 2dd2d09..0000000
--- a/solr/licenses/jetty-servlets-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-468c799c20b73de386b9de499ae1bb9cbbe7f559
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-servlets-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-servlets-9.3.8.v20160314.jar.sha1 b/solr/licenses/jetty-servlets-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..490ba2e
--- /dev/null
+++ b/solr/licenses/jetty-servlets-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+7c6cca49412e873cc2cee9903e3209525175f60d
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-util-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-util-9.3.6.v20151106.jar.sha1 b/solr/licenses/jetty-util-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index 48b6b12..0000000
--- a/solr/licenses/jetty-util-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8721c8e670c11ea19005c567733453956b6243fc
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-util-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-util-9.3.8.v20160314.jar.sha1 b/solr/licenses/jetty-util-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..026b5d0
--- /dev/null
+++ b/solr/licenses/jetty-util-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+01d53c7a7e7715e67d6f4edec6c5b328ee162e65
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-webapp-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-webapp-9.3.6.v20151106.jar.sha1 b/solr/licenses/jetty-webapp-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index 0c9b5ea..0000000
--- a/solr/licenses/jetty-webapp-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9cf00a3b7b2c1b6e024bb687e3719e1b0ff9e899
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-webapp-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-webapp-9.3.8.v20160314.jar.sha1 b/solr/licenses/jetty-webapp-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..e11683f
--- /dev/null
+++ b/solr/licenses/jetty-webapp-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+2f0dfef84af7c97f2a1f14db65aa3f37349420e4
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-xml-9.3.6.v20151106.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-xml-9.3.6.v20151106.jar.sha1 b/solr/licenses/jetty-xml-9.3.6.v20151106.jar.sha1
deleted file mode 100644
index 8bc22f3..0000000
--- a/solr/licenses/jetty-xml-9.3.6.v20151106.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b22e22977ea6c08751f8c945bb0785c35f9db28a
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/jetty-xml-9.3.8.v20160314.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jetty-xml-9.3.8.v20160314.jar.sha1 b/solr/licenses/jetty-xml-9.3.8.v20160314.jar.sha1
new file mode 100644
index 0000000..8e4d333
--- /dev/null
+++ b/solr/licenses/jetty-xml-9.3.8.v20160314.jar.sha1
@@ -0,0 +1 @@
+f02bbbf71d7ea706a95fedf7e76c3ff243049bfc
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ebf6153/solr/licenses/start.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/start.jar.sha1 b/solr/licenses/start.jar.sha1
index f67c9f5..e4c0d55 100644
--- a/solr/licenses/start.jar.sha1
+++ b/solr/licenses/start.jar.sha1
@@ -1 +1 @@
-1ea60b0f4e5f31b19c58471a6a616bac6699d75d
+365649a3404c9baa5b0345b3375cd9698f3cc43d
[17/50] lucene-solr:jira/SOLR-445: SOLR-8835: fix faceting exception
(uif) on multi-valued numeric docValues
Posted by ho...@apache.org.
SOLR-8835: fix faceting exception (uif) on multi-valued numeric docValues
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/95f20c6f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/95f20c6f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/95f20c6f
Branch: refs/heads/jira/SOLR-445
Commit: 95f20c6f00966db0b16d3abeaf12a768da83366a
Parents: c8b06b6
Author: yonik <yo...@apache.org>
Authored: Mon Mar 14 11:33:16 2016 -0400
Committer: yonik <yo...@apache.org>
Committed: Mon Mar 14 11:33:16 2016 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 ++
.../apache/solr/search/facet/FacetField.java | 13 ++---
.../solr/search/facet/TestJsonFacets.java | 50 ++++++++++++++++----
3 files changed, 49 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/95f20c6f/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index bb36297..2014020 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -294,6 +294,9 @@ Bug Fixes
* SOLR-8804: Fix a race condition in the ClusterStatus API call whereby the call would fail when a concurrent delete
collection api command was executed (Alexey Serba, Varun Thacker)
+* SOLR-8835: JSON Facet API: fix faceting exception on multi-valued numeric fields that
+ have docValues. (yonik)
+
Optimizations
----------------------
* SOLR-7876: Speed up queries and operations that use many terms when timeAllowed has not been
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/95f20c6f/solr/core/src/java/org/apache/solr/search/facet/FacetField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetField.java b/solr/core/src/java/org/apache/solr/search/facet/FacetField.java
index a1f1131..ce7f919 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetField.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetField.java
@@ -125,13 +125,8 @@ public class FacetField extends FacetRequest {
org.apache.lucene.document.FieldType.LegacyNumericType ntype = ft.getNumericType();
- if (sf.hasDocValues() && ntype==null) {
- // single and multi-valued string docValues
- return new FacetFieldProcessorDV(fcontext, this, sf);
- }
-
if (!multiToken) {
- if (sf.getType().getNumericType() != null) {
+ if (ntype != null) {
// single valued numeric (docvalues or fieldcache)
return new FacetFieldProcessorNumeric(fcontext, this, sf);
} else {
@@ -140,8 +135,10 @@ public class FacetField extends FacetRequest {
}
}
- // multivalued but field doesn't have docValues
- if (method == FacetMethod.DV) {
+ // multi-valued after this point
+
+ if (sf.hasDocValues() || method == FacetMethod.DV) {
+ // single and multi-valued string docValues
return new FacetFieldProcessorDV(fcontext, this, sf);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/95f20c6f/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
index 83220ed..8aaede1 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
@@ -343,25 +343,27 @@ public class TestJsonFacets extends SolrTestCaseHS {
doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_s", "cat_s","cat_s", "where_s","where_s", "num_d","num_d", "num_i","num_i", "super_s","super_s", "val_b","val_b", "date","date_dt", "sparse_s","sparse_s" ,"multi_ss","multi_ss") );
// multi-valued strings, long/float substitute for int/double
- doStatsTemplated(client, params(p, "facet","true", "rows","0", "noexist","noexist_ss", "cat_s","cat_ss", "where_s","where_ss", "num_d","num_f", "num_i","num_l", "super_s","super_ss", "val_b","val_b", "date","date_dt", "sparse_s","sparse_ss", "multi_ss","multi_ss") );
+ doStatsTemplated(client, params(p, "facet","true", "rows","0", "noexist","noexist_ss", "cat_s","cat_ss", "where_s","where_ss", "num_d","num_f", "num_i","num_l", "num_is","num_ls", "num_fs", "num_ds", "super_s","super_ss", "val_b","val_b", "date","date_dt", "sparse_s","sparse_ss", "multi_ss","multi_ss") );
// multi-valued strings, method=dv for terms facets
doStatsTemplated(client, params(p, "terms", "method:dv,", "rows", "0", "noexist", "noexist_ss", "cat_s", "cat_ss", "where_s", "where_ss", "num_d", "num_f", "num_i", "num_l", "super_s", "super_ss", "val_b", "val_b", "date", "date_dt", "sparse_s", "sparse_ss", "multi_ss", "multi_ss"));
// single valued docvalues for strings, and single valued numeric doc values for numeric fields
- doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sd", "cat_s","cat_sd", "where_s","where_sd", "num_d","num_dd", "num_i","num_id", "super_s","super_sd", "val_b","val_b", "date","date_dtd", "sparse_s","sparse_sd" ,"multi_ss","multi_sds") );
+ doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sd", "cat_s","cat_sd", "where_s","where_sd", "num_d","num_dd", "num_i","num_id", "num_is","num_lds", "num_fs","num_dds", "super_s","super_sd", "val_b","val_b", "date","date_dtd", "sparse_s","sparse_sd" ,"multi_ss","multi_sds") );
// multi-valued docvalues
FacetFieldProcessorDV.unwrap_singleValued_multiDv = false; // better multi-valued coverage
- doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sds", "cat_s","cat_sds", "where_s","where_sds", "num_d","num_d", "num_i","num_i", "super_s","super_sds", "val_b","val_b", "date","date_dtds", "sparse_s","sparse_sds" ,"multi_ss","multi_sds") );
+ doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sds", "cat_s","cat_sds", "where_s","where_sds", "num_d","num_d", "num_i","num_i", "num_is","num_ids", "num_fs","num_fds", "super_s","super_sds", "val_b","val_b", "date","date_dtds", "sparse_s","sparse_sds" ,"multi_ss","multi_sds") );
// multi-valued docvalues
FacetFieldProcessorDV.unwrap_singleValued_multiDv = true;
- doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sds", "cat_s","cat_sds", "where_s","where_sds", "num_d","num_d", "num_i","num_i", "super_s","super_sds", "val_b","val_b", "date","date_dtds", "sparse_s","sparse_sds" ,"multi_ss","multi_sds") );
+ doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sds", "cat_s","cat_sds", "where_s","where_sds", "num_d","num_d", "num_i","num_i", "num_is","num_ids", "num_fs","num_fds", "super_s","super_sds", "val_b","val_b", "date","date_dtds", "sparse_s","sparse_sds" ,"multi_ss","multi_sds") );
}
public static void doStatsTemplated(Client client, ModifiableSolrParams p) throws Exception {
p.set("Z_num_i", "Z_" + p.get("num_i") );
+ if (p.get("num_is") == null) p.add("num_is","num_is");
+ if (p.get("num_fs") == null) p.add("num_fs","num_fs");
MacroExpander m = new MacroExpander( p.getMap() );
@@ -369,6 +371,8 @@ public class TestJsonFacets extends SolrTestCaseHS {
String where_s = m.expand("${where_s}");
String num_d = m.expand("${num_d}");
String num_i = m.expand("${num_i}");
+ String num_is = m.expand("${num_is}");
+ String num_fs = m.expand("${num_fs}");
String Z_num_i = m.expand("${Z_num_i}");
String val_b = m.expand("${val_b}");
String date = m.expand("${date}");
@@ -379,17 +383,17 @@ public class TestJsonFacets extends SolrTestCaseHS {
client.deleteByQuery("*:*", null);
SolrInputDocument doc =
- sdoc("id", "1", cat_s, "A", where_s, "NY", num_d, "4", num_i, "2", super_s, "zodiac", date, "2001-01-01T01:01:01Z", val_b, "true", sparse_s, "one");
+ sdoc("id", "1", cat_s, "A", where_s, "NY", num_d, "4", num_i, "2", num_is,"2",num_is,"-5", num_fs,"2",num_fs,"-5", super_s, "zodiac", date, "2001-01-01T01:01:01Z", val_b, "true", sparse_s, "one");
client.add(doc, null);
client.add(doc, null);
client.add(doc, null); // a couple of deleted docs
- client.add(sdoc("id", "2", cat_s, "B", where_s, "NJ", num_d, "-9", num_i, "-5", super_s,"superman", date,"2002-02-02T02:02:02Z", val_b, "false" , multi_ss,"a", multi_ss,"b" , Z_num_i, "0"), null);
+ client.add(sdoc("id", "2", cat_s, "B", where_s, "NJ", num_d, "-9", num_i, "-5", num_is,"3",num_is,"-1", num_fs,"3",num_fs,"-1.5", super_s,"superman", date,"2002-02-02T02:02:02Z", val_b, "false" , multi_ss,"a", multi_ss,"b" , Z_num_i, "0"), null);
client.add(sdoc("id", "3"), null);
client.commit();
- client.add(sdoc("id", "4", cat_s, "A", where_s, "NJ", num_d, "2", num_i, "3", super_s,"spiderman", date,"2003-03-03T03:03:03Z" , multi_ss, "b", Z_num_i, ""+Integer.MIN_VALUE), null);
- client.add(sdoc("id", "5", cat_s, "B", where_s, "NJ", num_d, "11", num_i, "7", super_s,"batman" , date,"2001-02-03T01:02:03Z" ,sparse_s,"two", multi_ss, "a"), null);
+ client.add(sdoc("id", "4", cat_s, "A", where_s, "NJ", num_d, "2", num_i, "3", num_is,"0",num_is,"3", num_fs,"0", num_fs,"3", super_s,"spiderman", date,"2003-03-03T03:03:03Z" , multi_ss, "b", Z_num_i, ""+Integer.MIN_VALUE), null);
+ client.add(sdoc("id", "5", cat_s, "B", where_s, "NJ", num_d, "11", num_i, "7", num_is,"0", num_fs,"0", super_s,"batman" , date,"2001-02-03T01:02:03Z" ,sparse_s,"two", multi_ss, "a"), null);
client.commit();
- client.add(sdoc("id", "6", cat_s, "B", where_s, "NY", num_d, "-5", num_i, "-5", super_s,"hulk" , date,"2002-03-01T03:02:01Z" , multi_ss, "b", multi_ss, "a", Z_num_i, ""+Integer.MAX_VALUE), null);
+ client.add(sdoc("id", "6", cat_s, "B", where_s, "NY", num_d, "-5", num_i, "-5", num_is,"-1", num_fs,"-1.5", super_s,"hulk" , date,"2002-03-01T03:02:01Z" , multi_ss, "b", multi_ss, "a", Z_num_i, ""+Integer.MAX_VALUE), null);
client.commit();
// test for presence of debugging info
@@ -1097,6 +1101,34 @@ public class TestJsonFacets extends SolrTestCaseHS {
);
+
+ // multi-valued integer
+ client.testJQ(params(p, "q", "*:*"
+ , "json.facet", "{ " +
+ " c1:'unique(${num_is})', c2:'hll(${num_is})'" +
+ ",f1:{${terms} type:terms, field:${num_is} } " +
+ "}"
+ )
+ , "facets=={ count:6 " +
+ ", c1:5, c2:5" +
+ ", f1:{ buckets:[ {val:-1,count:2},{val:0,count:2},{val:3,count:2},{val:-5,count:1},{val:2,count:1} ] } " +
+ "} "
+ );
+
+ // multi-valued float
+ client.testJQ(params(p, "q", "*:*"
+ , "json.facet", "{ " +
+ " c1:'unique(${num_fs})', c2:'hll(${num_fs})'" +
+ ",f1:{${terms} type:terms, field:${num_fs} } " +
+ "}"
+ )
+ , "facets=={ count:6 " +
+ ", c1:5, c2:5" +
+ ", f1:{ buckets:[ {val:-1.5,count:2},{val:0.0,count:2},{val:3.0,count:2},{val:-5.0,count:1},{val:2.0,count:1} ] } " +
+ "} "
+ );
+
+
}
[25/50] lucene-solr:jira/SOLR-445: SOLR-7739 - applied patch from
Alessandro Benedetti for integrating Lucene classification into Solr
Posted by ho...@apache.org.
SOLR-7739 - applied patch from Alessandro Benedetti for integrating Lucene classification into Solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5801caab
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5801caab
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5801caab
Branch: refs/heads/jira/SOLR-445
Commit: 5801caab6c5fb881a590e06401096ea04111d905
Parents: 56ca641
Author: Tommaso Teofili <te...@adobe.com>
Authored: Tue Mar 15 12:29:07 2016 +0100
Committer: Tommaso Teofili <te...@adobe.com>
Committed: Tue Mar 15 12:29:07 2016 +0100
----------------------------------------------------------------------
dev-tools/idea/solr/core/src/java/solr-core.iml | 1 +
.../idea/solr/core/src/solr-core-tests.iml | 1 +
solr/common-build.xml | 4 +-
.../ClassificationUpdateProcessor.java | 102 ++++++++
.../ClassificationUpdateProcessorFactory.java | 223 ++++++++++++++++++
.../collection1/conf/schema-classification.xml | 43 ++++
.../conf/solrconfig-classification.xml | 53 +++++
...lassificationUpdateProcessorFactoryTest.java | 234 +++++++++++++++++++
8 files changed, 660 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5801caab/dev-tools/idea/solr/core/src/java/solr-core.iml
----------------------------------------------------------------------
diff --git a/dev-tools/idea/solr/core/src/java/solr-core.iml b/dev-tools/idea/solr/core/src/java/solr-core.iml
index f03268c..822b24f 100644
--- a/dev-tools/idea/solr/core/src/java/solr-core.iml
+++ b/dev-tools/idea/solr/core/src/java/solr-core.iml
@@ -27,6 +27,7 @@
<orderEntry type="module" module-name="expressions" />
<orderEntry type="module" module-name="analysis-common" />
<orderEntry type="module" module-name="lucene-core" />
+ <orderEntry type="module" module-name="classification" />
<orderEntry type="module" module-name="queryparser" />
<orderEntry type="module" module-name="join" />
<orderEntry type="module" module-name="sandbox" />
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5801caab/dev-tools/idea/solr/core/src/solr-core-tests.iml
----------------------------------------------------------------------
diff --git a/dev-tools/idea/solr/core/src/solr-core-tests.iml b/dev-tools/idea/solr/core/src/solr-core-tests.iml
index c9f722a..56f768b 100644
--- a/dev-tools/idea/solr/core/src/solr-core-tests.iml
+++ b/dev-tools/idea/solr/core/src/solr-core-tests.iml
@@ -21,6 +21,7 @@
<orderEntry type="module" scope="TEST" module-name="solr-core" />
<orderEntry type="module" scope="TEST" module-name="solrj" />
<orderEntry type="module" scope="TEST" module-name="lucene-core" />
+ <orderEntry type="module" scope="TEST" module-name="classification" />
<orderEntry type="module" scope="TEST" module-name="analysis-common" />
<orderEntry type="module" scope="TEST" module-name="queryparser" />
<orderEntry type="module" scope="TEST" module-name="queries" />
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5801caab/solr/common-build.xml
----------------------------------------------------------------------
diff --git a/solr/common-build.xml b/solr/common-build.xml
index 6a06928..78e10aa 100644
--- a/solr/common-build.xml
+++ b/solr/common-build.xml
@@ -108,6 +108,7 @@
<pathelement location="${queryparser.jar}"/>
<pathelement location="${join.jar}"/>
<pathelement location="${sandbox.jar}"/>
+ <pathelement location="${classification.jar}"/>
</path>
<path id="solr.base.classpath">
@@ -169,7 +170,7 @@
<target name="prep-lucene-jars"
depends="jar-lucene-core, jar-backward-codecs, jar-analyzers-phonetic, jar-analyzers-kuromoji, jar-codecs,jar-expressions, jar-suggest, jar-highlighter, jar-memory,
- jar-misc, jar-spatial-extras, jar-grouping, jar-queries, jar-queryparser, jar-join, jar-sandbox">
+ jar-misc, jar-spatial-extras, jar-grouping, jar-queries, jar-queryparser, jar-join, jar-sandbox, jar-classification">
<property name="solr.deps.compiled" value="true"/>
</target>
@@ -322,6 +323,7 @@
<link offline="true" href="${lucene.javadoc.url}highlighter" packagelistloc="${lucenedocs}/highlighter"/>
<link offline="true" href="${lucene.javadoc.url}memory" packagelistloc="${lucenedocs}/memory"/>
<link offline="true" href="${lucene.javadoc.url}misc" packagelistloc="${lucenedocs}/misc"/>
+ <link offline="true" href="${lucene.javadoc.url}classification" packagelistloc="${lucenedocs}/classification"/>
<link offline="true" href="${lucene.javadoc.url}spatial-extras" packagelistloc="${lucenedocs}/spatial-extras"/>
<links/>
<link href=""/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5801caab/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessor.java
new file mode 100644
index 0000000..b752565
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessor.java
@@ -0,0 +1,102 @@
+package org.apache.solr.update.processor;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.classification.ClassificationResult;
+import org.apache.lucene.classification.document.DocumentClassifier;
+import org.apache.lucene.classification.document.KNearestNeighborDocumentClassifier;
+import org.apache.lucene.classification.document.SimpleNaiveBayesDocumentClassifier;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.update.AddUpdateCommand;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This Class is a Request Update Processor to classify the document in input and add a field
+ * containing the class to the Document.
+ * It uses the Lucene Document Classification module, see {@link DocumentClassifier}.
+ */
+class ClassificationUpdateProcessor
+ extends UpdateRequestProcessor {
+
+ private String classFieldName; // the field to index the assigned class
+
+ private DocumentClassifier<BytesRef> classifier;
+
+ /**
+ * Sole constructor
+ *
+ * @param inputFieldNames fields to be used as classifier's inputs
+ * @param classFieldName field to be used as classifier's output
+ * @param minDf setting for {@link org.apache.lucene.queries.mlt.MoreLikeThis#minDocFreq}, in case algorithm is {@code "knn"}
+ * @param minTf setting for {@link org.apache.lucene.queries.mlt.MoreLikeThis#minTermFreq}, in case algorithm is {@code "knn"}
+ * @param k setting for k nearest neighbors to analyze, in case algorithm is {@code "knn"}
+ * @param algorithm the name of the classifier to use
+ * @param next next update processor in the chain
+ * @param indexReader index reader
+ * @param schema schema
+ */
+ public ClassificationUpdateProcessor(String[] inputFieldNames, String classFieldName, int minDf, int minTf, int k, String algorithm,
+ UpdateRequestProcessor next, LeafReader indexReader, IndexSchema schema) {
+ super(next);
+ this.classFieldName = classFieldName;
+ Map<String, Analyzer> field2analyzer = new HashMap<String, Analyzer>();
+ for (String fieldName : inputFieldNames) {
+ SchemaField fieldFromSolrSchema = schema.getField(fieldName);
+ Analyzer indexAnalyzer = fieldFromSolrSchema.getType().getQueryAnalyzer();
+ field2analyzer.put(fieldName, indexAnalyzer);
+ }
+ switch (algorithm) {
+ case "knn":
+ classifier = new KNearestNeighborDocumentClassifier(indexReader, null, null, k, minDf, minTf, classFieldName, field2analyzer, inputFieldNames);
+ break;
+ case "bayes":
+ classifier = new SimpleNaiveBayesDocumentClassifier(indexReader, null, classFieldName, field2analyzer, inputFieldNames);
+ break;
+ }
+ }
+
+ /**
+ * @param cmd the update command in input conaining the Document to classify
+ * @throws IOException If there is a low-level I/O error
+ */
+ @Override
+ public void processAdd(AddUpdateCommand cmd)
+ throws IOException {
+ SolrInputDocument doc = cmd.getSolrInputDocument();
+ Document luceneDocument = cmd.getLuceneDocument();
+ String assignedClass;
+ Object documentClass = doc.getFieldValue(classFieldName);
+ if (documentClass == null) {
+ ClassificationResult<BytesRef> classificationResult = classifier.assignClass(luceneDocument);
+ if (classificationResult != null) {
+ assignedClass = classificationResult.getAssignedClass().utf8ToString();
+ doc.addField(classFieldName, assignedClass);
+ }
+ }
+ super.processAdd(cmd);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5801caab/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessorFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessorFactory.java
new file mode 100644
index 0000000..79b3240
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessorFactory.java
@@ -0,0 +1,223 @@
+package org.apache.solr.update.processor;
+
+import org.apache.lucene.index.LeafReader;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.IndexSchema;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This class implements an UpdateProcessorFactory for the Classification Update Processor.
+ * It takes in input a series of parameter that will be necessary to instantiate and use the Classifier
+ */
+public class ClassificationUpdateProcessorFactory extends UpdateRequestProcessorFactory {
+
+ // Update Processor Config params
+ private static final String INPUT_FIELDS_PARAM = "inputFields";
+ private static final String CLASS_FIELD_PARAM = "classField";
+ private static final String ALGORITHM_PARAM = "algorithm";
+ private static final String KNN_MIN_TF_PARAM = "knn.minTf";
+ private static final String KNN_MIN_DF_PARAM = "knn.minDf";
+ private static final String KNN_K_PARAM = "knn.k";
+
+ //Update Processor Defaults
+ private static final int DEFAULT_MIN_TF = 1;
+ private static final int DEFAULT_MIN_DF = 1;
+ private static final int DEFAULT_K = 10;
+ private static final String DEFAULT_ALGORITHM = "knn";
+
+ private String[] inputFieldNames; // the array of fields to be sent to the Classifier
+
+ private String classFieldName; // the field containing the class for the Document
+
+ private String algorithm; // the Classification Algorithm to use - currently 'knn' or 'bayes'
+
+ private int minTf; // knn specific - the minimum Term Frequency for considering a term
+
+ private int minDf; // knn specific - the minimum Document Frequency for considering a term
+
+ private int k; // knn specific - thw window of top results to evaluate, when assgning the class
+
+ @Override
+ public void init(final NamedList args) {
+ if (args != null) {
+ SolrParams params = SolrParams.toSolrParams(args);
+
+ String fieldNames = params.get(INPUT_FIELDS_PARAM);// must be a comma separated list of fields
+ checkNotNull(INPUT_FIELDS_PARAM, fieldNames);
+ inputFieldNames = fieldNames.split("\\,");
+
+ classFieldName = params.get(CLASS_FIELD_PARAM);
+ checkNotNull(CLASS_FIELD_PARAM, classFieldName);
+
+ algorithm = params.get(ALGORITHM_PARAM);
+ if (algorithm == null)
+ algorithm = DEFAULT_ALGORITHM;
+
+ minTf = getIntParam(params, KNN_MIN_TF_PARAM, DEFAULT_MIN_TF);
+ minDf = getIntParam(params, KNN_MIN_DF_PARAM, DEFAULT_MIN_DF);
+ k = getIntParam(params, KNN_K_PARAM, DEFAULT_K);
+ }
+ }
+
+ /*
+ * Returns an Int parsed param or a default if the param is null
+ *
+ * @param params Solr params in input
+ * @param name the param name
+ * @param defaultValue the param default
+ * @return the Int value for the param
+ */
+ private int getIntParam(SolrParams params, String name, int defaultValue) {
+ String paramString = params.get(name);
+ int paramInt;
+ if (paramString != null && !paramString.isEmpty()) {
+ paramInt = Integer.parseInt(paramString);
+ } else {
+ paramInt = defaultValue;
+ }
+ return paramInt;
+ }
+
+ private void checkNotNull(String paramName, Object param) {
+ if (param == null) {
+ throw new SolrException
+ (SolrException.ErrorCode.SERVER_ERROR,
+ "Classification UpdateProcessor '" + paramName + "' can not be null");
+ }
+ }
+
+ @Override
+ public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ IndexSchema schema = req.getSchema();
+ LeafReader leafReader = req.getSearcher().getLeafReader();
+ return new ClassificationUpdateProcessor(inputFieldNames, classFieldName, minDf, minTf, k, algorithm, next, leafReader, schema);
+ }
+
+ /**
+ * get field names used as classifier's inputs
+ *
+ * @return the input field names
+ */
+ public String[] getInputFieldNames() {
+ return inputFieldNames;
+ }
+
+ /**
+ * set field names used as classifier's inputs
+ *
+ * @param inputFieldNames the input field names
+ */
+ public void setInputFieldNames(String[] inputFieldNames) {
+ this.inputFieldNames = inputFieldNames;
+ }
+
+ /**
+ * get field names used as classifier's output
+ *
+ * @return the output field name
+ */
+ public String getClassFieldName() {
+ return classFieldName;
+ }
+
+ /**
+ * set field names used as classifier's output
+ *
+ * @param classFieldName the output field name
+ */
+ public void setClassFieldName(String classFieldName) {
+ this.classFieldName = classFieldName;
+ }
+
+ /**
+ * get the name of the classifier algorithm used
+ *
+ * @return the classifier algorithm used
+ */
+ public String getAlgorithm() {
+ return algorithm;
+ }
+
+ /**
+ * set the name of the classifier algorithm used
+ *
+ * @param algorithm the classifier algorithm used
+ */
+ public void setAlgorithm(String algorithm) {
+ this.algorithm = algorithm;
+ }
+
+ /**
+ * get the min term frequency value to be used in case algorithm is {@code "knn"}
+ *
+ * @return the min term frequency
+ */
+ public int getMinTf() {
+ return minTf;
+ }
+
+ /**
+ * set the min term frequency value to be used in case algorithm is {@code "knn"}
+ *
+ * @param minTf the min term frequency
+ */
+ public void setMinTf(int minTf) {
+ this.minTf = minTf;
+ }
+
+ /**
+ * get the min document frequency value to be used in case algorithm is {@code "knn"}
+ *
+ * @return the min document frequency
+ */
+ public int getMinDf() {
+ return minDf;
+ }
+
+ /**
+ * set the min document frequency value to be used in case algorithm is {@code "knn"}
+ *
+ * @param minDf the min document frequency
+ */
+ public void setMinDf(int minDf) {
+ this.minDf = minDf;
+ }
+
+ /**
+ * get the the no. of nearest neighbor to analyze, to be used in case algorithm is {@code "knn"}
+ *
+ * @return the no. of neighbors to analyze
+ */
+ public int getK() {
+ return k;
+ }
+
+ /**
+ * set the the no. of nearest neighbor to analyze, to be used in case algorithm is {@code "knn"}
+ *
+ * @param k the no. of neighbors to analyze
+ */
+ public void setK(int k) {
+ this.k = k;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5801caab/solr/core/src/test-files/solr/collection1/conf/schema-classification.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-classification.xml b/solr/core/src/test-files/solr/collection1/conf/schema-classification.xml
new file mode 100644
index 0000000..89c27a6
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-classification.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ Test Schema for a simple Classification Test
+ -->
+<schema name="classification" version="1.1">
+ <fieldType name="string" class="solr.StrField"/>
+ <fieldType name="text_en" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.EnglishPossessiveFilterFactory"/>
+ <filter class="solr.PorterStemFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.EnglishPossessiveFilterFactory"/>
+ <filter class="solr.PorterStemFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <field name="id" type="string" indexed="true" stored="true" required="true"/>
+ <field name="title" type="text_en" indexed="true" stored="true"/>
+ <field name="content" type="text_en" indexed="true" stored="true"/>
+ <field name="author" type="string" indexed="true" stored="true"/>
+ <field name="cat" type="string" indexed="true" stored="true"/>
+ <uniqueKey>id</uniqueKey>
+</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5801caab/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml
new file mode 100644
index 0000000..3656335
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Test Config for a simple Classification Update Request Processor Chain
+ -->
+<config>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+ <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="solrconfig.snippet.randomindexconfig.xml"/>
+ <requestHandler name="standard" class="solr.StandardRequestHandler"></requestHandler>
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <updateLog enable="${enable.update.log:true}">
+ <str name="dir">${solr.ulog.dir:}</str>
+ </updateLog>
+
+ <commitWithin>
+ <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+ </commitWithin>
+
+ </updateHandler>
+
+ <updateRequestProcessorChain name="classification">
+ <processor class="solr.ClassificationUpdateProcessorFactory">
+ <str name="inputFields">title,content,author</str>
+ <str name="classField">cat</str>
+ <!-- Knn algorithm specific-->
+ <str name="algorithm">knn</str>
+ <str name="knn.minTf">1</str>
+ <str name="knn.minDf">1</str>
+ <str name="knn.k">5</str>
+ </processor>
+ <processor class="solr.RunUpdateProcessorFactory"/>
+ </updateRequestProcessorChain>
+</config>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5801caab/solr/core/src/test/org/apache/solr/update/processor/ClassificationUpdateProcessorFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/ClassificationUpdateProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/ClassificationUpdateProcessorFactoryTest.java
new file mode 100644
index 0000000..27d8dca
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/update/processor/ClassificationUpdateProcessorFactoryTest.java
@@ -0,0 +1,234 @@
+package org.apache.solr.update.processor;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.MultiMapSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.handler.UpdateRequestHandler;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests for {@link ClassificationUpdateProcessor} and {@link ClassificationUpdateProcessorFactory}
+ */
+public class ClassificationUpdateProcessorFactoryTest extends SolrTestCaseJ4 {
+ // field names are used in accordance with the solrconfig and schema supplied
+ private static final String ID = "id";
+ private static final String TITLE = "title";
+ private static final String CONTENT = "content";
+ private static final String AUTHOR = "author";
+ private static final String CLASS = "cat";
+
+ private static final String CHAIN = "classification";
+
+
+ private ClassificationUpdateProcessorFactory cFactoryToTest = new ClassificationUpdateProcessorFactory();
+ private NamedList args = new NamedList<String>();
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ System.setProperty("enable.update.log", "false");
+ initCore("solrconfig-classification.xml", "schema-classification.xml");
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ clearIndex();
+ assertU(commit());
+ }
+
+ @Before
+ public void initArgs() {
+ args.add("inputFields", "inputField1,inputField2");
+ args.add("classField", "classField1");
+ args.add("algorithm", "bayes");
+ args.add("knn.k", "9");
+ args.add("knn.minDf", "8");
+ args.add("knn.minTf", "10");
+ }
+
+ @Test
+ public void testFullInit() {
+ cFactoryToTest.init(args);
+
+ String[] inputFieldNames = cFactoryToTest.getInputFieldNames();
+ assertEquals("inputField1", inputFieldNames[0]);
+ assertEquals("inputField2", inputFieldNames[1]);
+ assertEquals("classField1", cFactoryToTest.getClassFieldName());
+ assertEquals("bayes", cFactoryToTest.getAlgorithm());
+ assertEquals(8, cFactoryToTest.getMinDf());
+ assertEquals(10, cFactoryToTest.getMinTf());
+ assertEquals(9, cFactoryToTest.getK());
+
+ }
+
+ @Test
+ public void testInitEmptyInputField() {
+ args.removeAll("inputFields");
+ try {
+ cFactoryToTest.init(args);
+ } catch (SolrException e) {
+ assertEquals("Classification UpdateProcessor 'inputFields' can not be null", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testInitEmptyClassField() {
+ args.removeAll("classField");
+ try {
+ cFactoryToTest.init(args);
+ } catch (SolrException e) {
+ assertEquals("Classification UpdateProcessor 'classField' can not be null", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testDefaults() {
+ args.removeAll("algorithm");
+ args.removeAll("knn.k");
+ args.removeAll("knn.minDf");
+ args.removeAll("knn.minTf");
+ cFactoryToTest.init(args);
+ assertEquals("knn", cFactoryToTest.getAlgorithm());
+ assertEquals(1, cFactoryToTest.getMinDf());
+ assertEquals(1, cFactoryToTest.getMinTf());
+ assertEquals(10, cFactoryToTest.getK());
+ }
+
+ @Test
+ public void testBasicClassification() throws Exception {
+ prepareTrainedIndex();
+ // To be classified,we index documents without a class and verify the expected one is returned
+ addDoc(adoc(ID, "10",
+ TITLE, "word4 word4 word4",
+ CONTENT, "word5 word5 ",
+ AUTHOR, "Name1 Surname1"));
+ addDoc(adoc(ID, "11",
+ TITLE, "word1 word1",
+ CONTENT, "word2 word2",
+ AUTHOR, "Name Surname"));
+ addDoc(commit());
+
+ Document doc10 = getDoc("10");
+ assertEquals("class2", doc10.get(CLASS));
+ Document doc11 = getDoc("11");
+ assertEquals("class1", doc11.get(CLASS));
+ }
+
+ /**
+ * Index some example documents with a class manually assigned.
+ * This will be our trained model.
+ *
+ * @throws Exception If there is a low-level I/O error
+ */
+ private void prepareTrainedIndex() throws Exception {
+ //class1
+ addDoc(adoc(ID, "1",
+ TITLE, "word1 word1 word1",
+ CONTENT, "word2 word2 word2",
+ AUTHOR, "Name Surname",
+ CLASS, "class1"));
+ addDoc(adoc(ID, "2",
+ TITLE, "word1 word1",
+ CONTENT, "word2 word2",
+ AUTHOR, "Name Surname",
+ CLASS, "class1"));
+ addDoc(adoc(ID, "3",
+ TITLE, "word1 word1 word1",
+ CONTENT, "word2",
+ AUTHOR, "Name Surname",
+ CLASS, "class1"));
+ addDoc(adoc(ID, "4",
+ TITLE, "word1 word1 word1",
+ CONTENT, "word2 word2 word2",
+ AUTHOR, "Name Surname",
+ CLASS, "class1"));
+ //class2
+ addDoc(adoc(ID, "5",
+ TITLE, "word4 word4 word4",
+ CONTENT, "word5 word5",
+ AUTHOR, "Name1 Surname1",
+ CLASS, "class2"));
+ addDoc(adoc(ID, "6",
+ TITLE, "word4 word4",
+ CONTENT, "word5",
+ AUTHOR, "Name1 Surname1",
+ CLASS, "class2"));
+ addDoc(adoc(ID, "7",
+ TITLE, "word4 word4 word4",
+ CONTENT, "word5 word5 word5",
+ AUTHOR, "Name1 Surname1",
+ CLASS, "class2"));
+ addDoc(adoc(ID, "8",
+ TITLE, "word4",
+ CONTENT, "word5 word5 word5 word5",
+ AUTHOR, "Name1 Surname1",
+ CLASS, "class2"));
+ addDoc(commit());
+ }
+
+ private Document getDoc(String id) throws IOException {
+ try (SolrQueryRequest req = req()) {
+ SolrIndexSearcher searcher = req.getSearcher();
+ TermQuery query = new TermQuery(new Term(ID, id));
+ TopDocs doc1 = searcher.search(query, 1);
+ ScoreDoc scoreDoc = doc1.scoreDocs[0];
+ return searcher.doc(scoreDoc.doc);
+ }
+ }
+
+ static void addDoc(String doc) throws Exception {
+ Map<String, String[]> params = new HashMap<>();
+ MultiMapSolrParams mmparams = new MultiMapSolrParams(params);
+ params.put(UpdateParams.UPDATE_CHAIN, new String[]{CHAIN});
+ SolrQueryRequestBase req = new SolrQueryRequestBase(h.getCore(),
+ (SolrParams) mmparams) {
+ };
+
+ UpdateRequestHandler handler = new UpdateRequestHandler();
+ handler.init(null);
+ ArrayList<ContentStream> streams = new ArrayList<>(2);
+ streams.add(new ContentStreamBase.StringStream(doc));
+ req.setContentStreams(streams);
+ handler.handleRequestBody(req, new SolrQueryResponse());
+ req.close();
+ }
+}
[50/50] lucene-solr:jira/SOLR-445: SOLR-445: hardent the
ToleratedUpdateError API to hide implementation details
Posted by ho...@apache.org.
SOLR-445: hardent the ToleratedUpdateError API to hide implementation details
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/21c0fe69
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/21c0fe69
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/21c0fe69
Branch: refs/heads/jira/SOLR-445
Commit: 21c0fe690dc4e968e484ee906632a50bf0273786
Parents: 6ec8c63
Author: Chris Hostetter <ho...@apache.org>
Authored: Sun Mar 20 16:44:17 2016 -0700
Committer: Chris Hostetter <ho...@apache.org>
Committed: Sun Mar 20 16:44:17 2016 -0700
----------------------------------------------------------------------
.../processor/TolerantUpdateProcessor.java | 4 +--
.../cloud/TestTolerantUpdateProcessorCloud.java | 10 +++---
.../solr/common/ToleratedUpdateError.java | 33 +++++++++++++-------
.../solr/common/TestToleratedUpdateError.java | 6 ++--
4 files changed, 31 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21c0fe69/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java
index 78457bb..9f9ff5e 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java
@@ -200,7 +200,7 @@ public class TolerantUpdateProcessor extends UpdateRequestProcessor {
// if we're lucky enough to get an immediate local failure (ie: we're a leader, or some other processor
// failed) then recording the multiple failures is a good thing -- helps us with an accurate fail
// fast if we exceed maxErrors
- if (CmdType.DELQ.equals(err.type)) {
+ if (CmdType.DELQ.equals(err.getType())) {
knownDBQErrors.add(err);
}
@@ -287,7 +287,7 @@ public class TolerantUpdateProcessor extends UpdateRequestProcessor {
continue;
}
- if (CmdType.DELQ.equals(err.type)) {
+ if (CmdType.DELQ.equals(err.getType())) {
if (knownDBQErrors.contains(err)) {
// we've already seen this identical error, probably a dup from another shard
continue;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21c0fe69/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java
index c47f221..41ff4af 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java
@@ -562,9 +562,9 @@ public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase {
actualKnownErrsCount, actualKnownErrs.size());
for (ToleratedUpdateError err : actualKnownErrs) {
assertEquals("only expected type of error is ADD: " + err,
- CmdType.ADD, err.type);
+ CmdType.ADD, err.getType());
assertTrue("failed err msg didn't match expected value: " + err,
- err.errorValue.contains("bogus_val"));
+ err.getMessage().contains("bogus_val"));
}
}
assertEquals(0, client.commit().getStatus()); // need to force since update didn't finish
@@ -636,11 +636,11 @@ public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase {
actualKnownErrsCount, actualKnownErrs.size());
for (ToleratedUpdateError err : actualKnownErrs) {
assertEquals("only expected type of error is ADD: " + err,
- CmdType.ADD, err.type);
+ CmdType.ADD, err.getType());
assertTrue("failed id had unexpected prefix: " + err,
- err.id.startsWith(S_TWO_PRE));
+ err.getId().startsWith(S_TWO_PRE));
assertTrue("failed err msg didn't match expected value: " + err,
- err.errorValue.contains("bogus_val"));
+ err.getMessage().contains("bogus_val"));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21c0fe69/solr/solrj/src/java/org/apache/solr/common/ToleratedUpdateError.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/ToleratedUpdateError.java b/solr/solrj/src/java/org/apache/solr/common/ToleratedUpdateError.java
index 6da6fc5..fd8b8c7 100644
--- a/solr/solrj/src/java/org/apache/solr/common/ToleratedUpdateError.java
+++ b/solr/solrj/src/java/org/apache/solr/common/ToleratedUpdateError.java
@@ -105,22 +105,31 @@ public final class ToleratedUpdateError {
metadataKey.substring(typeEnd+1), metadataVal);
}
- // nocommit: make these private & provide getter methods
- public final CmdType type;
- public final String id; // may be null depending on type
- public final String errorValue; // nocommit: refactor: rename message?
+ private final CmdType type;
+ private final String id;
+ private final String message;
- public ToleratedUpdateError(CmdType type, String id, String errorValue) {
- this.type = type;
+ public ToleratedUpdateError(CmdType type, String id, String message) {
assert null != type;
+ this.type = type;
assert null != id;
this.id = id;
- assert null != errorValue;
- this.errorValue = errorValue;
+ assert null != message;
+ this.message = message;
}
+ public CmdType getType() {
+ return type;
+ }
+ public String getId() {
+ return id;
+ }
+ public String getMessage() {
+ return message;
+ }
+
/**
* returns a string suitable for use as a key in {@link SolrException#setMetadata}
*
@@ -136,7 +145,7 @@ public final class ToleratedUpdateError {
* @see #parseMetadataIfToleratedUpdateError
*/
public String getMetadataValue() {
- return errorValue.toString();
+ return message.toString();
}
/**
@@ -148,7 +157,7 @@ public final class ToleratedUpdateError {
SimpleOrderedMap<String> entry = new SimpleOrderedMap<String>();
entry.add("type", type.toString());
entry.add("id", id);
- entry.add("message", errorValue);
+ entry.add("message", message);
return entry;
}
@@ -160,7 +169,7 @@ public final class ToleratedUpdateError {
int h = this.getClass().hashCode();
h = h * 31 + type.hashCode();
h = h * 31 + id.hashCode();
- h = h * 31 + errorValue.hashCode();
+ h = h * 31 + message.hashCode();
return h;
}
@@ -169,7 +178,7 @@ public final class ToleratedUpdateError {
ToleratedUpdateError that = (ToleratedUpdateError)o;
return that.type.equals(this.type)
&& that.id.equals(this.id)
- && that.errorValue.equals(this.errorValue);
+ && that.message.equals(this.message);
}
return false;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21c0fe69/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java b/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java
index a76443a..aba07ae 100644
--- a/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java
+++ b/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java
@@ -155,9 +155,9 @@ public class TestToleratedUpdateError extends LuceneTestCase {
}
public void compare(ToleratedUpdateError in, ToleratedUpdateError out) {
- assertEquals(out.type, in.type);
- assertEquals(out.id, in.id);
- assertEquals(out.errorValue, in.errorValue);
+ assertEquals(out.getType(), in.getType());
+ assertEquals(out.getId(), in.getId());
+ assertEquals(out.getMessage(), in.getMessage());
assertEquals(out.hashCode(), in.hashCode());
assertEquals(out.toString(), in.toString());
[32/50] lucene-solr:jira/SOLR-445: LUCENE-7109:
LatLonPoint.newPolygonQuery should use two-phase iterator
Posted by ho...@apache.org.
LUCENE-7109: LatLonPoint.newPolygonQuery should use two-phase iterator
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6ea458a0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6ea458a0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6ea458a0
Branch: refs/heads/jira/SOLR-445
Commit: 6ea458a0edaa4b2e30a2c31dcb703350ee3936c1
Parents: 0f78235
Author: Robert Muir <rm...@apache.org>
Authored: Wed Mar 16 09:25:06 2016 -0400
Committer: Robert Muir <rm...@apache.org>
Committed: Wed Mar 16 09:26:15 2016 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../document/LatLonPointInPolygonQuery.java | 65 +++++++++++++++++---
.../document/TestLatLonPointInPolygonQuery.java | 49 +++++++++++++++
3 files changed, 107 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ea458a0/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 3b4f507..be28e34 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -23,6 +23,9 @@ Optimizations
* LUCENE-7105: Optimize LatLonPoint's newDistanceQuery. (Robert Muir)
+* LUCENE-7109: LatLonPoint's newPolygonQuery supports two-phase
+ iteration. (Robert Muir)
+
* LUCENE-7097: IntroSorter now recurses to 2 * log_2(count) quicksort
stack depth before switching to heapsort (Adrien Grand, Mike McCandless)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ea458a0/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java
index fb9189f..cb98895 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java
@@ -23,15 +23,23 @@ import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.search.Weight;
import org.apache.lucene.index.PointValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.DocIdSetBuilder;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.SparseFixedBitSet;
import org.apache.lucene.spatial.util.GeoRelationUtils;
import org.apache.lucene.spatial.util.GeoUtils;
@@ -110,9 +118,6 @@ final class LatLonPointInPolygonQuery extends Query {
// I don't use RandomAccessWeight here: it's no good to approximate with "match all docs"; this is an inverted structure and should be
// used in the first pass:
- // TODO: except that the polygon verify is costly! The approximation should be all docs in all overlapping cells, and matches() should
- // then check the polygon
-
return new ConstantScoreWeight(this) {
@Override
@@ -130,22 +135,28 @@ final class LatLonPointInPolygonQuery extends Query {
}
LatLonPoint.checkCompatible(fieldInfo);
+ // approximation (postfiltering has not yet been applied)
DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc());
+ // subset of documents that need no postfiltering, this is purely an optimization
+ final BitSet preApproved;
+ // dumb heuristic: if the field is really sparse, use a sparse impl
+ if (values.getDocCount(field) * 100L < reader.maxDoc()) {
+ preApproved = new SparseFixedBitSet(reader.maxDoc());
+ } else {
+ preApproved = new FixedBitSet(reader.maxDoc());
+ }
values.intersect(field,
new IntersectVisitor() {
@Override
public void visit(int docID) {
result.add(docID);
+ preApproved.set(docID);
}
@Override
public void visit(int docID, byte[] packedValue) {
- assert packedValue.length == 8;
- double lat = LatLonPoint.decodeLatitude(packedValue, 0);
- double lon = LatLonPoint.decodeLongitude(packedValue, Integer.BYTES);
- if (GeoRelationUtils.pointInPolygon(polyLons, polyLats, lat, lon)) {
- result.add(docID);
- }
+ // TODO: range checks
+ result.add(docID);
}
@Override
@@ -172,7 +183,41 @@ final class LatLonPointInPolygonQuery extends Query {
}
});
- return new ConstantScoreScorer(this, score(), result.build().iterator());
+ DocIdSet set = result.build();
+ final DocIdSetIterator disi = set.iterator();
+ if (disi == null) {
+ return null;
+ }
+
+ // return two-phase iterator using docvalues to postfilter candidates
+ SortedNumericDocValues docValues = DocValues.getSortedNumeric(reader, field);
+ TwoPhaseIterator iterator = new TwoPhaseIterator(disi) {
+ @Override
+ public boolean matches() throws IOException {
+ int docId = disi.docID();
+ if (preApproved.get(docId)) {
+ return true;
+ } else {
+ docValues.setDocument(docId);
+ int count = docValues.count();
+ for (int i = 0; i < count; i++) {
+ long encoded = docValues.valueAt(i);
+ double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32));
+ double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF));
+ if (GeoRelationUtils.pointInPolygon(polyLons, polyLats, docLatitude, docLongitude)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+
+ @Override
+ public float matchCost() {
+ return 20 * polyLons.length; // TODO: make this fancier, but currently linear with number of vertices
+ }
+ };
+ return new ConstantScoreScorer(this, score(), iterator);
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ea458a0/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointInPolygonQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointInPolygonQuery.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointInPolygonQuery.java
new file mode 100644
index 0000000..de87027
--- /dev/null
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointInPolygonQuery.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.document;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+
+/** Simple tests for {@link LatLonPoint#newPolygonQuery} */
+public class TestLatLonPointInPolygonQuery extends LuceneTestCase {
+
+ /** test we can search for a polygon */
+ public void testBasics() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+
+ // add a doc with a point
+ Document document = new Document();
+ document.add(new LatLonPoint("field", 18.313694, -65.227444));
+ writer.addDocument(document);
+
+ // search and verify we found our doc
+ IndexReader reader = writer.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ assertEquals(1, searcher.count(LatLonPoint.newPolygonQuery("field",
+ new double[] { 18, 18, 19, 19, 18 },
+ new double[] { -66, -65, -65, -66, -66 })));
+
+ reader.close();
+ writer.close();
+ dir.close();
+ }
+}
[16/50] lucene-solr:jira/SOLR-445: fix test bug: this test expects
single segment index
Posted by ho...@apache.org.
fix test bug: this test expects single segment index
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c8b06b68
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c8b06b68
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c8b06b68
Branch: refs/heads/jira/SOLR-445
Commit: c8b06b68e6f8a6c50f8eeb0e8b7e7171164a9c20
Parents: f706c9d
Author: Mike McCandless <mi...@apache.org>
Authored: Mon Mar 14 10:42:03 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Mon Mar 14 10:42:03 2016 -0400
----------------------------------------------------------------------
.../test/org/apache/lucene/queries/payloads/TestPayloadSpans.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8b06b68/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java
index 9e9228b..2c0204d 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java
@@ -435,6 +435,7 @@ public class TestPayloadSpans extends LuceneTestCase {
writer.addDocument(doc);
}
+ writer.forceMerge(1);
closeIndexReader = writer.getReader();
writer.close();
[46/50] lucene-solr:jira/SOLR-445: SOLR-445: Merge branch 'master'
into jira/SOLR-445
Posted by ho...@apache.org.
SOLR-445: Merge branch 'master' into jira/SOLR-445
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8cc0a384
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8cc0a384
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8cc0a384
Branch: refs/heads/jira/SOLR-445
Commit: 8cc0a38453b389bdb031d78ad638b76dfa27f2d5
Parents: a0d48f8 a22099a
Author: Chris Hostetter <ho...@apache.org>
Authored: Thu Mar 17 10:37:54 2016 -0700
Committer: Chris Hostetter <ho...@apache.org>
Committed: Thu Mar 17 10:37:54 2016 -0700
----------------------------------------------------------------------
dev-tools/idea/solr/core/src/java/solr-core.iml | 1 +
.../idea/solr/core/src/solr-core-tests.iml | 1 +
lucene/CHANGES.txt | 30 +-
.../simpletext/SimpleTextPointsWriter.java | 8 +-
.../org/apache/lucene/codecs/PointsWriter.java | 14 +-
.../codecs/lucene60/Lucene60PointsWriter.java | 20 +-
.../org/apache/lucene/document/DoublePoint.java | 8 +-
.../org/apache/lucene/document/FloatPoint.java | 8 +-
.../org/apache/lucene/document/IntPoint.java | 8 +-
.../org/apache/lucene/document/LongPoint.java | 6 +-
.../org/apache/lucene/index/PointValues.java | 90 ++-
.../apache/lucene/index/PointValuesWriter.java | 16 +-
.../org/apache/lucene/util/IntroSorter.java | 6 +-
.../org/apache/lucene/util/OfflineSorter.java | 44 +-
.../org/apache/lucene/util/bkd/BKDWriter.java | 221 ++++---
.../apache/lucene/util/bkd/HeapPointReader.java | 17 +-
.../apache/lucene/util/bkd/HeapPointWriter.java | 47 +-
.../lucene/util/bkd/OfflinePointReader.java | 101 ++-
.../lucene/util/bkd/OfflinePointWriter.java | 21 +-
.../org/apache/lucene/util/bkd/PointReader.java | 37 +-
.../org/apache/lucene/index/Test2BPoints.java | 36 +-
.../apache/lucene/index/TestPointValues.java | 64 +-
.../apache/lucene/search/TestTermScorer.java | 2 +-
.../lucene/search/spans/TestSpanCollection.java | 2 +-
.../apache/lucene/util/TestOfflineSorter.java | 2 +-
.../apache/lucene/util/bkd/Test2BBKDPoints.java | 121 ++++
.../org/apache/lucene/util/bkd/TestBKD.java | 22 +-
lucene/ivy-versions.properties | 2 +-
.../jetty-continuation-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-continuation-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-http-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-http-9.3.8.v20160314.jar.sha1 | 1 +
.../licenses/jetty-io-9.3.6.v20151106.jar.sha1 | 1 -
.../licenses/jetty-io-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-server-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-server-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-servlet-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-servlet-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-util-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-util-9.3.8.v20160314.jar.sha1 | 1 +
.../apache/lucene/index/memory/MemoryIndex.java | 622 ++++++++++++++++---
.../lucene/index/memory/TestMemoryIndex.java | 293 +++++++++
.../memory/TestMemoryIndexAgainstRAMDir.java | 192 ++++++
.../queries/payloads/TestPayloadSpans.java | 11 +-
.../org/apache/lucene/document/LatLonPoint.java | 78 ++-
.../document/LatLonPointDistanceComparator.java | 213 +++++++
.../document/LatLonPointDistanceQuery.java | 148 ++++-
.../document/LatLonPointInPolygonQuery.java | 65 +-
.../lucene/document/LatLonPointSortField.java | 106 ++++
.../lucene/document/TestBigIntegerPoint.java | 4 +-
.../lucene/document/TestInetAddressPoint.java | 4 +-
.../apache/lucene/document/TestLatLonPoint.java | 15 +-
.../document/TestLatLonPointDistanceQuery.java | 4 +-
.../document/TestLatLonPointDistanceSort.java | 289 +++++++++
.../document/TestLatLonPointInPolygonQuery.java | 49 ++
solr/CHANGES.txt | 47 +-
solr/common-build.xml | 4 +-
.../src/java/org/apache/solr/core/SolrCore.java | 3 +-
.../apache/solr/handler/loader/JsonLoader.java | 6 +-
.../solr/response/GeoJSONResponseWriter.java | 345 ++++++++++
.../solr/response/JSONResponseWriter.java | 2 +-
.../transform/GeoTransformerFactory.java | 224 +++++++
.../response/transform/TransformerFactory.java | 1 +
.../response/transform/WriteableGeoJSON.java | 55 ++
.../solr/schema/AbstractSpatialFieldType.java | 87 ++-
.../org/apache/solr/schema/DateRangeField.java | 4 +-
.../apache/solr/search/SolrIndexSearcher.java | 4 +-
.../apache/solr/search/facet/FacetField.java | 13 +-
.../org/apache/solr/update/TransactionLog.java | 4 +-
.../ClassificationUpdateProcessor.java | 102 +++
.../ClassificationUpdateProcessorFactory.java | 223 +++++++
.../collection1/conf/schema-classification.xml | 43 ++
.../solr/collection1/conf/schema-spatial.xml | 1 +
.../conf/solrconfig-classification.xml | 53 ++
.../org/apache/solr/handler/JsonLoaderTest.java | 20 +
.../response/TestGeoJSONResponseWriter.java | 279 +++++++++
.../solr/schema/SpatialRPTFieldTypeTest.java | 38 +-
.../solr/schema/TestUseDocValuesAsStored.java | 157 +++--
.../solr/search/facet/TestJsonFacets.java | 50 +-
.../test/org/apache/solr/update/TestUpdate.java | 21 +
...lassificationUpdateProcessorFactoryTest.java | 234 +++++++
.../jetty-continuation-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-continuation-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-deploy-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-deploy-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-http-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-http-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/jetty-io-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-io-9.3.8.v20160314.jar.sha1 | 1 +
.../licenses/jetty-jmx-9.3.6.v20151106.jar.sha1 | 1 -
.../licenses/jetty-jmx-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-rewrite-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-rewrite-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-security-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-security-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-server-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-server-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-servlet-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-servlet-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-servlets-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-servlets-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-util-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-util-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-webapp-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-webapp-9.3.8.v20160314.jar.sha1 | 1 +
.../licenses/jetty-xml-9.3.6.v20151106.jar.sha1 | 1 -
.../licenses/jetty-xml-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/start.jar.sha1 | 2 +-
.../basic_configs/conf/managed-schema | 32 +-
.../conf/managed-schema | 53 +-
.../conf/managed-schema | 43 +-
.../org/apache/solr/common/cloud/DocRouter.java | 27 +-
.../apache/solr/common/util/JavaBinCodec.java | 4 +-
.../java/org/apache/solr/cloud/ChaosMonkey.java | 100 +--
114 files changed, 4735 insertions(+), 632 deletions(-)
----------------------------------------------------------------------
[22/50] lucene-solr:jira/SOLR-445: Change entry to reflect that
LUCENE-7087 has been backported to 6.0
Posted by ho...@apache.org.
Change entry to reflect that LUCENE-7087 has been backported to 6.0
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8185c8a1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8185c8a1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8185c8a1
Branch: refs/heads/jira/SOLR-445
Commit: 8185c8a11dbfd170b046e4239b0222b5b0bf2007
Parents: 02bb6c0
Author: Martijn van Groningen <ma...@gmail.com>
Authored: Tue Mar 15 09:49:17 2016 +0100
Committer: Martijn van Groningen <mv...@apache.org>
Committed: Tue Mar 15 09:49:17 2016 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8185c8a1/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4998eb0..10d4d10 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -24,11 +24,6 @@ Optimizations
* LUCENE-7097: IntroSorter now recurses to 2 * log_2(count) quicksort
stack depth before switching to heapsort (Adrien Grand, Mike McCandless)
-Other
-
-* LUCENE-7087: Let MemoryIndex#fromDocument(...) accept 'Iterable<? extends IndexableField>'
- as document instead of 'Document'. (Martijn van Groningen)
-
======================= Lucene 6.0.0 =======================
System Requirements
@@ -205,6 +200,9 @@ Other
* LUCENE-7035: Upgrade icu4j to 56.1/unicode 8. (Robert Muir)
+* LUCENE-7087: Let MemoryIndex#fromDocument(...) accept 'Iterable<? extends IndexableField>'
+ as document instead of 'Document'. (Martijn van Groningen)
+
======================= Lucene 5.5.0 =======================
New Features
[48/50] lucene-solr:jira/SOLR-445: SOLR-445: harden & add logging to
test
Posted by ho...@apache.org.
SOLR-445: harden & add logging to test
also rename since chaos monkey isn't going to be involved (due to SOLR-8872)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1aa1ba3b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1aa1ba3b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1aa1ba3b
Branch: refs/heads/jira/SOLR-445
Commit: 1aa1ba3b3af69cad65b7a411ca88e120a418a598
Parents: aeda8dc
Author: Chris Hostetter <ho...@apache.org>
Authored: Sun Mar 20 13:56:43 2016 -0700
Committer: Chris Hostetter <ho...@apache.org>
Committed: Sun Mar 20 13:56:43 2016 -0700
----------------------------------------------------------------------
.../TestTolerantUpdateProcessorChaosMonkey.java | 307 -----------------
.../TestTolerantUpdateProcessorRandomCloud.java | 330 +++++++++++++++++++
2 files changed, 330 insertions(+), 307 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1aa1ba3b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorChaosMonkey.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorChaosMonkey.java b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorChaosMonkey.java
deleted file mode 100644
index 25dbbd4..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorChaosMonkey.java
+++ /dev/null
@@ -1,307 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import java.io.File;
-import java.lang.invoke.MethodHandles;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.HashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.lucene.util.TestUtil;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.assertUpdateTolerantErrors;
-import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.addErr;
-import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.delIErr;
-import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.delQErr;
-import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.f;
-import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.update;
-import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.ExpectedErr;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.response.UpdateResponse;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.SolrInputField;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.ToleratedUpdateError;
-import org.apache.solr.common.ToleratedUpdateError.CmdType;
-import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.solr.util.RevertDefaultThreadHandlerRule;
-
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test of TolerantUpdateProcessor using a randomized MiniSolrCloud & ChaosMokney.
- * Reuses some utility methods in {@link TestTolerantUpdateProcessorCloud}
- *
- * <p>
- * <b>NOTE:</b> This test sets up a static instance of MiniSolrCloud with a single collection
- * and several clients pointed at specific nodes. These are all re-used across multiple test methods,
- * and assumes that the state of the cluster is healthy between tests.
- * </p>
- *
- */
-public class TestTolerantUpdateProcessorChaosMonkey extends SolrCloudTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final String COLLECTION_NAME = "test_col";
-
- /** A basic client for operations at the cloud level, default collection will be set */
- private static CloudSolrClient CLOUD_CLIENT;
-
- @BeforeClass
- private static void createMiniSolrCloudCluster() throws Exception {
-
- final String configName = "solrCloudCollectionConfig";
- final File configDir = new File(TEST_HOME() + File.separator + "collection1" + File.separator + "conf");
-
- final int numShards = TestUtil.nextInt(random(), 2, TEST_NIGHTLY ? 5 : 3);
- final int repFactor = TestUtil.nextInt(random(), 2, TEST_NIGHTLY ? 5 : 3);
- // at least one server won't have any replicas
- final int numServers = 1 + (numShards * repFactor);
-
- configureCluster(numServers)
- .addConfig(configName, configDir.toPath())
- .configure();
-
- Thread.sleep(2000); // anoying attempt to work arround SOLR-8862 // nocommit ? ? ?
-
- Map<String, String> collectionProperties = new HashMap<>();
- collectionProperties.put("config", "solrconfig-distrib-update-processor-chains.xml");
- collectionProperties.put("schema", "schema15.xml"); // string id
-
-
- assertNotNull(cluster.createCollection(COLLECTION_NAME, numShards, repFactor,
- configName, null, null, collectionProperties));
-
- CLOUD_CLIENT = cluster.getSolrClient();
- CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
-
- ZkStateReader zkStateReader = CLOUD_CLIENT.getZkStateReader();
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION_NAME, zkStateReader, true, true, 330);
-
- // nocommit: init chaos monkey
-
- }
-
- @AfterClass
- private static void cleanup() throws Exception {
- // nocommit: shutdown the monkey
- }
-
- @Before
- private void deleteAllDocs() throws Exception {
- assertEquals(0, update(params("commit","true")).deleteByQuery("*:*").process(CLOUD_CLIENT).getStatus());
- assertEquals("index should be empty",
- 0, CLOUD_CLIENT.query(params("q","*:*","rows","0")).getResults().getNumFound());
- }
-
- public void testRandomUpdatesWithChaos() throws Exception {
- final int maxDocId = atLeast(10000);
- final BitSet expectedDocIds = new BitSet(maxDocId+1);
-
- final int numIters = atLeast(50);
- for (int i = 0; i < numIters; i++) {
-
- // nocommit: chaosMonkey.causeSomeChaos()
-
- final UpdateRequest req = update(params("maxErrors","-1",
- "update.chain", "tolerant-chain-max-errors-10"));
- final int numCmds = TestUtil.nextInt(random(), 1, 20);
- final List<ExpectedErr> expectedErrors = new ArrayList<ExpectedErr>(numCmds);
- int expectedErrorsCount = 0;
- // it's ambigious/confusing which order mixed DELQ + ADD (or ADD and DELI for the same ID)
- // in the same request wll be processed by various clients, so we keep things simple
- // and ensure that no single doc Id is affected by more then one command in the same request
- final BitSet docsAffectedThisRequest = new BitSet(maxDocId+1);
- for (int cmdIter = 0; cmdIter < numCmds; cmdIter++) {
- if ((maxDocId / 2) < docsAffectedThisRequest.cardinality()) {
- // we're already mucking with more then half the docs in the index
- break;
- }
-
- final boolean causeError = random().nextBoolean();
- if (causeError) {
- expectedErrorsCount++;
- }
-
- if (random().nextBoolean()) {
- // add a doc
- final int id_i = randomUnsetBit(random(), docsAffectedThisRequest, maxDocId);
- final String id = "id_"+id_i;
- docsAffectedThisRequest.set(id_i);
- if (causeError) {
- expectedErrors.add(addErr(id));
- } else {
- expectedDocIds.set(id_i);
- }
- req.add(doc(f("id",id),
- f("id_i", id_i),
- f("foo_i", causeError ? "bogus_val" : TestUtil.nextInt(random(), 42, 666))));
-
- } else {
- // delete something
- if (random().nextBoolean()) {
- // delete by id
- final int id_i = randomUnsetBit(random(), docsAffectedThisRequest, maxDocId);
- final String id = "id_"+id_i;
- final boolean docExists = expectedDocIds.get(id_i);
- docsAffectedThisRequest.set(id_i);
- long versionConstraint = docExists ? 1 : -1;
- if (causeError) {
- versionConstraint = -1 * versionConstraint;
- expectedErrors.add(delIErr(id));
- } else {
- // if doc exists it will legitimately be deleted
- expectedDocIds.clear(id_i);
- }
- req.deleteById(id, versionConstraint);
-
- } else {
- // delete by query
- final String q;
- if (causeError) {
- // if our DBQ is going to fail, then it doesn't matter what q we use,
- // none of the docs in the index will be affected either way
- q = "foo_i:[42 TO ....giberish";
- expectedErrors.add(delQErr(q));
- } else {
- // ensure our DBQ is only over a range of docs not already affected
- // by any other cmds in this request
- final int rangeAxis = randomUnsetBit(random(), docsAffectedThisRequest, maxDocId);
- final int loBound = docsAffectedThisRequest.previousSetBit(rangeAxis);
- final int hiBound = docsAffectedThisRequest.nextSetBit(rangeAxis);
- final int lo = TestUtil.nextInt(random(), loBound+1, rangeAxis);
- final int hi = TestUtil.nextInt(random(), rangeAxis,
- // bound might be negative if no set bits above axis
- (hiBound < 0) ? maxDocId : hiBound-1);
-
- // NOTE: clear & set are exclusive of hi, so we use "}" in range query accordingly
- q = "id_i:[" + lo + " TO " + hi + (causeError ? "...giberish" : "}");
- expectedDocIds.clear(lo, hi);
- docsAffectedThisRequest.set(lo, hi);
- }
- req.deleteByQuery(q);
- }
- }
- }
- assertEquals("expected error count sanity check: " + req.toString(),
- expectedErrorsCount, expectedErrors.size());
-
- // nocommit: 50% randomly: use an HttpSolrClient from the list of servers the monkey says are up
- final SolrClient client = CLOUD_CLIENT;
-
- UpdateResponse rsp = req.process(client);
- assertUpdateTolerantErrors(expectedErrors.toString(), rsp,
- expectedErrors.toArray(new ExpectedErr[expectedErrors.size()]));
-
- }
- assertEquals("commit failed?", 0, CLOUD_CLIENT.commit().getStatus());
-
- assertEquals("final doc count doesn't match bitself cardinality",
- expectedDocIds.cardinality(),
- CLOUD_CLIENT.query(params("q","*:*","rows","0")).getResults().getNumFound());
- }
-
- /** sanity check that randomUnsetBit works as expected
- * @see #randomUnsetBit
- */
- public void testSanityRandomUnsetBit() {
- final int max = atLeast(100);
- BitSet bits = new BitSet(max+1);
- for (int i = 0; i <= max; i++) {
- assertFalse("how is bitset already full? iter="+i+" card="+bits.cardinality()+"/max="+max,
- bits.cardinality() == max+1);
- final int nextBit = randomUnsetBit(random(), bits, max);
- assertTrue("nextBit shouldn't be negative yet: " + nextBit,
- 0 <= nextBit);
- assertTrue("nextBit can't exceed max: " + nextBit,
- nextBit <= max);
- assertFalse("expect unset: " + nextBit, bits.get(nextBit));
- bits.set(nextBit);
- }
-
- assertEquals("why isn't bitset full?", max+1, bits.cardinality());
-
- final int firstClearBit = bits.nextClearBit(0);
- assertTrue("why is there a clear bit? = " + firstClearBit,
- max < firstClearBit);
- assertEquals("why is a bit set above max?",
- -1, bits.nextSetBit(max+1));
-
- assertEquals("wrong nextBit at end of all iters", -1,
- randomUnsetBit(random(), bits, max));
- assertEquals("wrong nextBit at redundent end of all iters", -1,
- randomUnsetBit(random(), bits, max));
- }
-
- public static SolrInputDocument doc(SolrInputField... fields) {
- // SolrTestCaseJ4 has same method name, prevents static import from working
- return TestTolerantUpdateProcessorCloud.doc(fields);
- }
-
- /**
- * Given a BitSet, returns a random bit that is currently false, or -1 if all bits are true.
- * NOTE: this method is not fair.
- */
- public static final int randomUnsetBit(Random r, BitSet bits, final int max) {
- // NOTE: don't forget, BitSet will grow automatically if not careful
- if (bits.cardinality() == max+1) {
- return -1;
- }
- final int candidate = TestUtil.nextInt(r, 0, max);
- if (bits.get(candidate)) {
- final int lo = bits.previousClearBit(candidate);
- final int hi = bits.nextClearBit(candidate);
- if (lo < 0 && max < hi) {
- fail("how the hell did we not short circut out? card="+bits.cardinality()+"/size="+bits.size());
- } else if (lo < 0) {
- return hi;
- } else if (max < hi) {
- return lo;
- } // else...
- return ((candidate - lo) < (hi - candidate)) ? lo : hi;
- }
- return candidate;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1aa1ba3b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java
new file mode 100644
index 0000000..536bb89
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java
@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cloud;
+
+import java.io.File;
+import java.lang.invoke.MethodHandles;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.assertUpdateTolerantErrors;
+import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.addErr;
+import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.delIErr;
+import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.delQErr;
+import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.f;
+import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.update;
+import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.ExpectedErr;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.SolrInputField;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.ToleratedUpdateError;
+import org.apache.solr.common.ToleratedUpdateError.CmdType;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.util.RevertDefaultThreadHandlerRule;
+
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test of TolerantUpdateProcessor using a randomized MiniSolrCloud.
+ * Reuses some utility methods in {@link TestTolerantUpdateProcessorCloud}
+ *
+ * <p>
+ * <b>NOTE:</b> This test sets up a static instance of MiniSolrCloud with a single collection
+ * and several clients pointed at specific nodes. These are all re-used across multiple test methods,
+ * and assumes that the state of the cluster is healthy between tests.
+ * </p>
+ *
+ */
+public class TestTolerantUpdateProcessorRandomCloud extends SolrCloudTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final String COLLECTION_NAME = "test_col";
+
+ /** A basic client for operations at the cloud level, default collection will be set */
+ private static CloudSolrClient CLOUD_CLIENT;
+ /** one HttpSolrClient for each server */
+ private static List<SolrClient> NODE_CLIENTS;
+
+ @BeforeClass
+ private static void createMiniSolrCloudCluster() throws Exception {
+
+ final String configName = "solrCloudCollectionConfig";
+ final File configDir = new File(TEST_HOME() + File.separator + "collection1" + File.separator + "conf");
+
+ final int numShards = TestUtil.nextInt(random(), 2, TEST_NIGHTLY ? 5 : 3);
+ final int repFactor = TestUtil.nextInt(random(), 2, TEST_NIGHTLY ? 5 : 3);
+ // at least one server won't have any replicas
+ final int numServers = 1 + (numShards * repFactor);
+
+ configureCluster(numServers)
+ .addConfig(configName, configDir.toPath())
+ .configure();
+
+ Thread.sleep(2000); // anoying attempt to work arround SOLR-8862 // nocommit ? ? ?
+
+ Map<String, String> collectionProperties = new HashMap<>();
+ collectionProperties.put("config", "solrconfig-distrib-update-processor-chains.xml");
+ collectionProperties.put("schema", "schema15.xml"); // string id
+
+
+ assertNotNull(cluster.createCollection(COLLECTION_NAME, numShards, repFactor,
+ configName, null, null, collectionProperties));
+
+ CLOUD_CLIENT = cluster.getSolrClient();
+ CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
+
+ NODE_CLIENTS = new ArrayList<SolrClient>(numServers);
+
+ for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+ URL jettyURL = jetty.getBaseUrl();
+ NODE_CLIENTS.add(new HttpSolrClient(jettyURL.toString() + "/" + COLLECTION_NAME + "/"));
+ }
+
+ ZkStateReader zkStateReader = CLOUD_CLIENT.getZkStateReader();
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION_NAME, zkStateReader, true, true, 330);
+
+ }
+
+ @Before
+ private void deleteAllDocs() throws Exception {
+ assertEquals(0, update(params("commit","true")).deleteByQuery("*:*").process(CLOUD_CLIENT).getStatus());
+ assertEquals("index should be empty", 0L, countDocs(CLOUD_CLIENT));
+ }
+
+ public void testRandomUpdates() throws Exception {
+ final int maxDocId = atLeast(10000);
+ final BitSet expectedDocIds = new BitSet(maxDocId+1);
+
+ final int numIters = atLeast(50);
+ for (int i = 0; i < numIters; i++) {
+
+ final UpdateRequest req = update(params("maxErrors","-1",
+ "update.chain", "tolerant-chain-max-errors-10"));
+ final int numCmds = TestUtil.nextInt(random(), 1, 20);
+ final List<ExpectedErr> expectedErrors = new ArrayList<ExpectedErr>(numCmds);
+ int expectedErrorsCount = 0;
+ // it's ambigious/confusing which order mixed DELQ + ADD (or ADD and DELI for the same ID)
+ // in the same request wll be processed by various clients, so we keep things simple
+ // and ensure that no single doc Id is affected by more then one command in the same request
+ final BitSet docsAffectedThisRequest = new BitSet(maxDocId+1);
+ for (int cmdIter = 0; cmdIter < numCmds; cmdIter++) {
+ if ((maxDocId / 2) < docsAffectedThisRequest.cardinality()) {
+ // we're already mucking with more then half the docs in the index
+ break;
+ }
+
+ final boolean causeError = random().nextBoolean();
+ if (causeError) {
+ expectedErrorsCount++;
+ }
+
+ if (random().nextBoolean()) {
+ // add a doc
+ final int id_i = randomUnsetBit(random(), docsAffectedThisRequest, maxDocId);
+ final String id = "id_"+id_i;
+ docsAffectedThisRequest.set(id_i);
+ if (causeError) {
+ expectedErrors.add(addErr(id));
+ } else {
+ expectedDocIds.set(id_i);
+ }
+ final String val = causeError ? "bogus_val" : (""+TestUtil.nextInt(random(), 42, 666));
+ req.add(doc(f("id",id),
+ f("id_i", id_i),
+ f("foo_i", val)));
+ log.info("ADD: {} = {}", id, val);
+ } else {
+ // delete something
+ if (random().nextBoolean()) {
+ // delete by id
+ final int id_i = randomUnsetBit(random(), docsAffectedThisRequest, maxDocId);
+ final String id = "id_"+id_i;
+ final boolean docExists = expectedDocIds.get(id_i);
+ docsAffectedThisRequest.set(id_i);
+ long versionConstraint = docExists ? 1 : -1;
+ if (causeError) {
+ versionConstraint = -1 * versionConstraint;
+ expectedErrors.add(delIErr(id));
+ } else {
+ // if doc exists it will legitimately be deleted
+ expectedDocIds.clear(id_i);
+ }
+ req.deleteById(id, versionConstraint);
+ log.info("DEL: {} = {}", id, causeError ? "ERR" : "OK" );
+ } else {
+ // delete by query
+ final String q;
+ if (causeError) {
+ // even though our DBQ is gibberish that's going to fail, record a docId as affected
+ // so that we don't generate the same random DBQ and get redundent errors
+ // (problematic because of how DUP forwarded DBQs have to have their errors deduped by TUP)
+ final int id_i = randomUnsetBit(random(), docsAffectedThisRequest, maxDocId);
+ docsAffectedThisRequest.set(id_i);
+ q = "foo_i:["+id_i+" TO ....giberish";
+ expectedErrors.add(delQErr(q));
+ } else {
+ // ensure our DBQ is only over a range of docs not already affected
+ // by any other cmds in this request
+ final int rangeAxis = randomUnsetBit(random(), docsAffectedThisRequest, maxDocId);
+ final int loBound = docsAffectedThisRequest.previousSetBit(rangeAxis);
+ final int hiBound = docsAffectedThisRequest.nextSetBit(rangeAxis);
+ final int lo = TestUtil.nextInt(random(), loBound+1, rangeAxis);
+ final int hi = TestUtil.nextInt(random(), rangeAxis,
+ // bound might be negative if no set bits above axis
+ (hiBound < 0) ? maxDocId : hiBound-1);
+
+ if (lo != hi) {
+ assert lo < hi : "lo="+lo+" hi="+hi;
+ // NOTE: clear & set are exclusive of hi, so we use "}" in range query accordingly
+ q = "id_i:[" + lo + " TO " + hi + "}";
+ expectedDocIds.clear(lo, hi);
+ docsAffectedThisRequest.set(lo, hi);
+ } else {
+ // edge case: special case DBQ of one doc
+ assert (lo == rangeAxis && hi == rangeAxis) : "lo="+lo+" axis="+rangeAxis+" hi="+hi;
+ q = "id_i:[" + lo + " TO " + lo + "]"; // have to be inclusive of both ends
+ expectedDocIds.clear(lo);
+ docsAffectedThisRequest.set(lo);
+ }
+ }
+ req.deleteByQuery(q);
+ log.info("DEL: {}", q);
+ }
+ }
+ }
+ assertEquals("expected error count sanity check: " + req.toString(),
+ expectedErrorsCount, expectedErrors.size());
+
+ final SolrClient client = random().nextBoolean() ? CLOUD_CLIENT
+ : NODE_CLIENTS.get(TestUtil.nextInt(random(), 0, NODE_CLIENTS.size()-1));
+
+ final UpdateResponse rsp = req.process(client);
+ assertUpdateTolerantErrors(client.toString() + " => " + expectedErrors.toString(), rsp,
+ expectedErrors.toArray(new ExpectedErr[expectedErrors.size()]));
+ log.info("END: {}", expectedDocIds.cardinality());
+
+ assertEquals("post update commit failed?", 0, CLOUD_CLIENT.commit().getStatus());
+
+ for (int j = 0; j < 5; j++) {
+ if (expectedDocIds.cardinality() == countDocs(CLOUD_CLIENT)) {
+ break;
+ }
+ log.info("sleeping to give searchers a chance to re-open #" + j);
+ Thread.sleep(200);
+ }
+ assertEquals("cloud client doc count doesn't match bitself cardinality",
+ expectedDocIds.cardinality(), countDocs(CLOUD_CLIENT));
+ }
+ }
+
+ /** sanity check that randomUnsetBit works as expected
+ * @see #randomUnsetBit
+ */
+ public void testSanityRandomUnsetBit() {
+ final int max = atLeast(100);
+ BitSet bits = new BitSet(max+1);
+ for (int i = 0; i <= max; i++) {
+ assertFalse("how is bitset already full? iter="+i+" card="+bits.cardinality()+"/max="+max,
+ bits.cardinality() == max+1);
+ final int nextBit = randomUnsetBit(random(), bits, max);
+ assertTrue("nextBit shouldn't be negative yet: " + nextBit,
+ 0 <= nextBit);
+ assertTrue("nextBit can't exceed max: " + nextBit,
+ nextBit <= max);
+ assertFalse("expect unset: " + nextBit, bits.get(nextBit));
+ bits.set(nextBit);
+ }
+
+ assertEquals("why isn't bitset full?", max+1, bits.cardinality());
+
+ final int firstClearBit = bits.nextClearBit(0);
+ assertTrue("why is there a clear bit? = " + firstClearBit,
+ max < firstClearBit);
+ assertEquals("why is a bit set above max?",
+ -1, bits.nextSetBit(max+1));
+
+ assertEquals("wrong nextBit at end of all iters", -1,
+ randomUnsetBit(random(), bits, max));
+ assertEquals("wrong nextBit at redundent end of all iters", -1,
+ randomUnsetBit(random(), bits, max));
+ }
+
+ public static SolrInputDocument doc(SolrInputField... fields) {
+ // SolrTestCaseJ4 has same method name, prevents static import from working
+ return TestTolerantUpdateProcessorCloud.doc(fields);
+ }
+
+ /**
+ * Given a BitSet, returns a random bit that is currently false, or -1 if all bits are true.
+ * NOTE: this method is not fair.
+ */
+ public static final int randomUnsetBit(Random r, BitSet bits, final int max) {
+ // NOTE: don't forget, BitSet will grow automatically if not careful
+ if (bits.cardinality() == max+1) {
+ return -1;
+ }
+ final int candidate = TestUtil.nextInt(r, 0, max);
+ if (bits.get(candidate)) {
+ final int lo = bits.previousClearBit(candidate);
+ final int hi = bits.nextClearBit(candidate);
+ if (lo < 0 && max < hi) {
+ fail("how the hell did we not short circut out? card="+bits.cardinality()+"/size="+bits.size());
+ } else if (lo < 0) {
+ return hi;
+ } else if (max < hi) {
+ return lo;
+ } // else...
+ return ((candidate - lo) < (hi - candidate)) ? lo : hi;
+ }
+ return candidate;
+ }
+
+ public static final long countDocs(SolrClient c) throws Exception {
+ return c.query(params("q","*:*","rows","0")).getResults().getNumFound();
+ }
+}
[47/50] lucene-solr:jira/SOLR-445: SOLR-445: fix test bugs,
and put in a stupid work around for SOLR-8862
Posted by ho...@apache.org.
SOLR-445: fix test bugs, and put in a stupid work around for SOLR-8862
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/aeda8dc4
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/aeda8dc4
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/aeda8dc4
Branch: refs/heads/jira/SOLR-445
Commit: aeda8dc4ae881c4ec405d70dcbf1d0b2c30871b7
Parents: 8cc0a38
Author: Chris Hostetter <ho...@apache.org>
Authored: Thu Mar 17 22:43:27 2016 -0700
Committer: Chris Hostetter <ho...@apache.org>
Committed: Thu Mar 17 22:43:27 2016 -0700
----------------------------------------------------------------------
.../TestTolerantUpdateProcessorChaosMonkey.java | 74 +++++++++++---------
1 file changed, 39 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/aeda8dc4/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorChaosMonkey.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorChaosMonkey.java b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorChaosMonkey.java
index 47ece84..25dbbd4 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorChaosMonkey.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorChaosMonkey.java
@@ -103,25 +103,13 @@ public class TestTolerantUpdateProcessorChaosMonkey extends SolrCloudTestCase {
.addConfig(configName, configDir.toPath())
.configure();
+ Thread.sleep(2000); // anoying attempt to work arround SOLR-8862 // nocommit ? ? ?
+
Map<String, String> collectionProperties = new HashMap<>();
collectionProperties.put("config", "solrconfig-distrib-update-processor-chains.xml");
collectionProperties.put("schema", "schema15.xml"); // string id
- // nocommit: SOLR-8862 why does this seed constantly cause createCollection to reliably complain that there are no live servers?!?!?!
- //
- // ant test -Dtestcase=TestTolerantUpdateProcessorChaosMonkey -Dtests.seed=E73756F21DF21ECC -Dtests.slow=true -Dtests.locale=tr-TR -Dtests.timezone=America/Manaus -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
- //
- //
- // [junit4] > Throwable #1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:43171/solr, http://127.0.0.1:49629/solr, http://127.0.0.1:59664/solr, http://127.0.0.1:40944/solr, http://127.0.0.1:38476/solr]
- // [junit4] > at __randomizedtesting.SeedInfo.seed([E73756F21DF21ECC]:0)
- // [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:352)
- // [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1157)
- // [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:927)
- // [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:863)
- // [junit4] > at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
- // [junit4] > at org.apache.solr.cloud.MiniSolrCloudCluster.makeCollectionsRequest(MiniSolrCloudCluster.java:415)
- // [junit4] > at org.apache.solr.cloud.MiniSolrCloudCluster.createCollection(MiniSolrCloudCluster.java:399)
- //
+
assertNotNull(cluster.createCollection(COLLECTION_NAME, numShards, repFactor,
configName, null, null, collectionProperties));
@@ -144,7 +132,7 @@ public class TestTolerantUpdateProcessorChaosMonkey extends SolrCloudTestCase {
private void deleteAllDocs() throws Exception {
assertEquals(0, update(params("commit","true")).deleteByQuery("*:*").process(CLOUD_CLIENT).getStatus());
assertEquals("index should be empty",
- CLOUD_CLIENT.query(params("q","*:*","rows","0")).getResults().getNumFound());
+ 0, CLOUD_CLIENT.query(params("q","*:*","rows","0")).getResults().getNumFound());
}
public void testRandomUpdatesWithChaos() throws Exception {
@@ -166,6 +154,11 @@ public class TestTolerantUpdateProcessorChaosMonkey extends SolrCloudTestCase {
// and ensure that no single doc Id is affected by more then one command in the same request
final BitSet docsAffectedThisRequest = new BitSet(maxDocId+1);
for (int cmdIter = 0; cmdIter < numCmds; cmdIter++) {
+ if ((maxDocId / 2) < docsAffectedThisRequest.cardinality()) {
+ // we're already mucking with more then half the docs in the index
+ break;
+ }
+
final boolean causeError = random().nextBoolean();
if (causeError) {
expectedErrorsCount++;
@@ -173,7 +166,7 @@ public class TestTolerantUpdateProcessorChaosMonkey extends SolrCloudTestCase {
if (random().nextBoolean()) {
// add a doc
- final int id_i = randomUnsetBit(random(), docsAffectedThisRequest);
+ final int id_i = randomUnsetBit(random(), docsAffectedThisRequest, maxDocId);
final String id = "id_"+id_i;
docsAffectedThisRequest.set(id_i);
if (causeError) {
@@ -189,7 +182,7 @@ public class TestTolerantUpdateProcessorChaosMonkey extends SolrCloudTestCase {
// delete something
if (random().nextBoolean()) {
// delete by id
- final int id_i = randomUnsetBit(random(), docsAffectedThisRequest);
+ final int id_i = randomUnsetBit(random(), docsAffectedThisRequest, maxDocId);
final String id = "id_"+id_i;
final boolean docExists = expectedDocIds.get(id_i);
docsAffectedThisRequest.set(id_i);
@@ -214,7 +207,7 @@ public class TestTolerantUpdateProcessorChaosMonkey extends SolrCloudTestCase {
} else {
// ensure our DBQ is only over a range of docs not already affected
// by any other cmds in this request
- final int rangeAxis = randomUnsetBit(random(), docsAffectedThisRequest);
+ final int rangeAxis = randomUnsetBit(random(), docsAffectedThisRequest, maxDocId);
final int loBound = docsAffectedThisRequest.previousSetBit(rangeAxis);
final int hiBound = docsAffectedThisRequest.nextSetBit(rangeAxis);
final int lo = TestUtil.nextInt(random(), loBound+1, rangeAxis);
@@ -253,22 +246,32 @@ public class TestTolerantUpdateProcessorChaosMonkey extends SolrCloudTestCase {
* @see #randomUnsetBit
*/
public void testSanityRandomUnsetBit() {
- BitSet bits = new BitSet(atLeast(100));
- int nextBit = -2;
- for (int i = 0; i < bits.size(); i++) {
- assertFalse("how is bitset already full? iter="+i+" card="+bits.cardinality()+"/size="+bits.size(),
- bits.cardinality() == bits.size());
- nextBit = randomUnsetBit(random(), bits);
+ final int max = atLeast(100);
+ BitSet bits = new BitSet(max+1);
+ for (int i = 0; i <= max; i++) {
+ assertFalse("how is bitset already full? iter="+i+" card="+bits.cardinality()+"/max="+max,
+ bits.cardinality() == max+1);
+ final int nextBit = randomUnsetBit(random(), bits, max);
assertTrue("nextBit shouldn't be negative yet: " + nextBit,
0 <= nextBit);
+ assertTrue("nextBit can't exceed max: " + nextBit,
+ nextBit <= max);
assertFalse("expect unset: " + nextBit, bits.get(nextBit));
bits.set(nextBit);
}
- assertEquals("why isn't bitset full?", bits.size(), bits.cardinality());
- nextBit = randomUnsetBit(random(), bits);
- assertEquals("wrong nextBit at end of all iters", -1, nextBit);
- nextBit = randomUnsetBit(random(), bits);
- assertEquals("wrong nextBit at redundent end of all iters", -1, nextBit);
+
+ assertEquals("why isn't bitset full?", max+1, bits.cardinality());
+
+ final int firstClearBit = bits.nextClearBit(0);
+ assertTrue("why is there a clear bit? = " + firstClearBit,
+ max < firstClearBit);
+ assertEquals("why is a bit set above max?",
+ -1, bits.nextSetBit(max+1));
+
+ assertEquals("wrong nextBit at end of all iters", -1,
+ randomUnsetBit(random(), bits, max));
+ assertEquals("wrong nextBit at redundent end of all iters", -1,
+ randomUnsetBit(random(), bits, max));
}
public static SolrInputDocument doc(SolrInputField... fields) {
@@ -280,19 +283,20 @@ public class TestTolerantUpdateProcessorChaosMonkey extends SolrCloudTestCase {
* Given a BitSet, returns a random bit that is currently false, or -1 if all bits are true.
* NOTE: this method is not fair.
*/
- public static final int randomUnsetBit(Random r, BitSet bits) {
- if (bits.cardinality() == bits.size()) {
+ public static final int randomUnsetBit(Random r, BitSet bits, final int max) {
+ // NOTE: don't forget, BitSet will grow automatically if not careful
+ if (bits.cardinality() == max+1) {
return -1;
}
- final int candidate = TestUtil.nextInt(r, 0, bits.size()-1);
+ final int candidate = TestUtil.nextInt(r, 0, max);
if (bits.get(candidate)) {
final int lo = bits.previousClearBit(candidate);
final int hi = bits.nextClearBit(candidate);
- if (lo < 0 && bits.size() <= hi) {
+ if (lo < 0 && max < hi) {
fail("how the hell did we not short circut out? card="+bits.cardinality()+"/size="+bits.size());
} else if (lo < 0) {
return hi;
- } else if (bits.size() <= hi) {
+ } else if (max < hi) {
return lo;
} // else...
return ((candidate - lo) < (hi - candidate)) ? lo : hi;
[30/50] lucene-solr:jira/SOLR-445: SOLR-8836: Return 400,
and a SolrException when an invalid json is provided to the update
handler instead of 500.
Posted by ho...@apache.org.
SOLR-8836: Return 400, and a SolrException when an invalid json is provided to the update handler instead of 500.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/30a77b73
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/30a77b73
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/30a77b73
Branch: refs/heads/jira/SOLR-445
Commit: 30a77b73b603ba52a50da397aefc6f9a88f05732
Parents: 870baaf
Author: anshum <an...@apache.org>
Authored: Tue Mar 15 10:55:03 2016 -0700
Committer: anshum <an...@apache.org>
Committed: Tue Mar 15 10:55:20 2016 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../apache/solr/handler/loader/JsonLoader.java | 6 +++++-
.../org/apache/solr/handler/JsonLoaderTest.java | 20 ++++++++++++++++++++
3 files changed, 28 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/30a77b73/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 2014020..691e87f 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -425,6 +425,9 @@ Other Changes
* SOLR-8799: Improve error message when tuple can't be read by SolrJ JDBC (Kevin Risden, Joel Bernstein)
+* SOLR-8836: Return 400, and a SolrException when an invalid json is provided to the update handler
+ instead of 500. (Jason Gerlowski via Anshum Gupta)
+
================== 5.5.1 ==================
Bug Fixes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/30a77b73/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
index e99b243..ba800ff 100644
--- a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
+++ b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
@@ -50,6 +50,7 @@ import org.apache.solr.update.RollbackUpdateCommand;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.util.RecordingJSONParser;
import org.noggit.JSONParser;
+import org.noggit.JSONParser.ParseException;
import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -111,7 +112,10 @@ public class JsonLoader extends ContentStreamLoader {
}
this.processUpdate(reader);
- } finally {
+ } catch (ParseException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cannot parse provided JSON: " + e.getMessage());
+ }
+ finally {
IOUtils.closeQuietly(reader);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/30a77b73/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java b/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java
index f2316da..e27ca1a 100644
--- a/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java
@@ -181,6 +181,26 @@ public class JsonLoaderTest extends SolrTestCaseJ4 {
req.close();
}
+ @Test
+ public void testInvalidJsonProducesBadRequestSolrException() throws Exception
+ {
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ BufferingRequestProcessor p = new BufferingRequestProcessor(null);
+ JsonLoader loader = new JsonLoader();
+ String invalidJsonString = "}{";
+
+ try(SolrQueryRequest req = req()) {
+ try {
+ loader.load(req, rsp, new ContentStreamBase.StringStream(invalidJsonString), p);
+ fail("Expected invalid JSON to produce a SolrException.");
+ } catch (SolrException expectedException) {
+ assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, expectedException.code());
+ assertTrue(expectedException.getMessage().contains("Cannot parse"));
+ assertTrue(expectedException.getMessage().contains("JSON"));
+ }
+ }
+ }
+
public void testSimpleFormatInAdd() throws Exception
{
String str = "{'add':[{'id':'1'},{'id':'2'}]}".replace('\'', '"');
[45/50] lucene-solr:jira/SOLR-445: SOLR-8866: UpdateLog now throws an
error if it can't serialize a field value
Posted by ho...@apache.org.
SOLR-8866: UpdateLog now throws an error if it can't serialize a field value
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a22099a3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a22099a3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a22099a3
Branch: refs/heads/jira/SOLR-445
Commit: a22099a3986de1f36f926b4e106827c5308708b0
Parents: c1e95d7
Author: David Smiley <ds...@apache.org>
Authored: Thu Mar 17 13:22:16 2016 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Thu Mar 17 13:22:16 2016 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../org/apache/solr/update/TransactionLog.java | 4 +++-
.../test/org/apache/solr/update/TestUpdate.java | 21 ++++++++++++++++++++
.../apache/solr/common/util/JavaBinCodec.java | 4 +++-
4 files changed, 30 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a22099a3/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 1be92c9..3863ceb 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -66,6 +66,9 @@ Other Changes
* SOLR-8860: Remove back-compat handling of router format made in SOLR-4221 in 4.5.0. (shalin)
+* SOLR-8866: UpdateLog will now throw an exception if it doesn't know how to serialize a value.
+ (David Smiley)
+
================== 6.0.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a22099a3/solr/core/src/java/org/apache/solr/update/TransactionLog.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/TransactionLog.java b/solr/core/src/java/org/apache/solr/update/TransactionLog.java
index 474bcaf..673d683 100644
--- a/solr/core/src/java/org/apache/solr/update/TransactionLog.java
+++ b/solr/core/src/java/org/apache/solr/update/TransactionLog.java
@@ -95,7 +95,9 @@ public class TransactionLog implements Closeable {
codec.writeByteArray(br.bytes, br.offset, br.length);
return null;
}
- return o;
+ // Fallback: we have no idea how to serialize this. Be noisy to prevent insidious bugs
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "TransactionLog doesn't know how to serialize " + o.getClass() + "; try implementing ObjectResolver?");
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a22099a3/solr/core/src/test/org/apache/solr/update/TestUpdate.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/TestUpdate.java b/solr/core/src/test/org/apache/solr/update/TestUpdate.java
index 381231f..13a2479 100644
--- a/solr/core/src/test/org/apache/solr/update/TestUpdate.java
+++ b/solr/core/src/test/org/apache/solr/update/TestUpdate.java
@@ -18,9 +18,11 @@ package org.apache.solr.update;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.io.IOException;
import java.util.concurrent.Callable;
public class TestUpdate extends SolrTestCaseJ4 {
@@ -243,4 +245,23 @@ public class TestUpdate extends SolrTestCaseJ4 {
}
+ @Test // SOLR-8866
+ public void testUpdateLogThrowsForUnknownTypes() throws IOException {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "444");
+ doc.addField("text", new Object());//Object shouldn't be serialized later...
+
+ AddUpdateCommand cmd = new AddUpdateCommand(req());
+ cmd.solrDoc = doc;
+ try {
+ h.getCore().getUpdateHandler().addDoc(cmd); // should throw
+ } catch (SolrException e) {
+ if (e.getMessage().contains("serialize")) {
+ return;//passed test
+ }
+ throw e;
+ }
+ fail();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a22099a3/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java b/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
index 63c1b28..fe9ad08 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
@@ -207,7 +207,9 @@ public class JavaBinCodec {
if (writeKnownType(tmpVal)) return;
}
}
-
+ // Fallback to do *something*.
+ // note: if the user of this codec doesn't want this (e.g. UpdateLog) it can supply an ObjectResolver that does
+ // something else like throw an exception.
writeVal(val.getClass().getName() + ':' + val.toString());
}
[29/50] lucene-solr:jira/SOLR-445: LUCENE-7105: Optimize
LatLonPoint.newDistanceQuery
Posted by ho...@apache.org.
LUCENE-7105: Optimize LatLonPoint.newDistanceQuery
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/870baafc
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/870baafc
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/870baafc
Branch: refs/heads/jira/SOLR-445
Commit: 870baafc82b0853349db55b7886a6f31b54a69d5
Parents: 3ba7456
Author: Robert Muir <rm...@apache.org>
Authored: Tue Mar 15 11:17:52 2016 -0400
Committer: Robert Muir <rm...@apache.org>
Committed: Tue Mar 15 11:18:15 2016 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 2 +
.../document/LatLonPointDistanceQuery.java | 88 ++++++++++++++++----
2 files changed, 75 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/870baafc/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 59cd092..3b4f507 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -21,6 +21,8 @@ Optimizations
* LUCENE-7099: LatLonPoint's newDistanceQuery supports two-phase
iteration. (Robert Muir)
+* LUCENE-7105: Optimize LatLonPoint's newDistanceQuery. (Robert Muir)
+
* LUCENE-7097: IntroSorter now recurses to 2 * log_2(count) quicksort
stack depth before switching to heapsort (Adrien Grand, Mike McCandless)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/870baafc/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
index 3f86f1e..e8c7c08 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
@@ -41,7 +41,9 @@ import org.apache.lucene.spatial.util.GeoUtils;
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.DocIdSetBuilder;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.SparseFixedBitSet;
+import org.apache.lucene.util.StringHelper;
/**
* Distance query for {@link LatLonPoint}.
@@ -74,16 +76,41 @@ final class LatLonPointDistanceQuery extends Query {
@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
GeoRect box = GeoUtils.circleToBBox(longitude, latitude, radiusMeters);
- final GeoRect box1;
- final GeoRect box2;
+ // create bounding box(es) for the distance range
+ // these are pre-encoded with LatLonPoint's encoding
+ final byte minLat[] = new byte[Integer.BYTES];
+ final byte maxLat[] = new byte[Integer.BYTES];
+ final byte minLon[] = new byte[Integer.BYTES];
+ final byte maxLon[] = new byte[Integer.BYTES];
+ // second set of longitude ranges to check (for cross-dateline case)
+ final byte minLon2[] = new byte[Integer.BYTES];
+
+ NumericUtils.intToSortableBytes(LatLonPoint.encodeLatitude(box.minLat), minLat, 0);
+ NumericUtils.intToSortableBytes(LatLonPoint.encodeLatitude(box.maxLat), maxLat, 0);
// crosses dateline: split
if (box.crossesDateline()) {
- box1 = new GeoRect(-180.0, box.maxLon, box.minLat, box.maxLat);
- box2 = new GeoRect(box.minLon, 180.0, box.minLat, box.maxLat);
+ // box1
+ NumericUtils.intToSortableBytes(Integer.MIN_VALUE, minLon, 0);
+ NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.maxLon), maxLon, 0);
+ // box2
+ NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.minLon), minLon2, 0);
+ } else {
+ NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.minLon), minLon, 0);
+ NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.maxLon), maxLon, 0);
+ // disable box2
+ NumericUtils.intToSortableBytes(Integer.MAX_VALUE, minLon2, 0);
+ }
+
+ // compute a maximum partial haversin: unless our box is crazy, we can use this bound
+ // to reject edge cases faster in matches()
+ final double minPartialDistance;
+ if (box.maxLon - longitude < 90 && longitude - box.minLon < 90) {
+ minPartialDistance = Math.max(LatLonPointDistanceComparator.haversin1(latitude, longitude, latitude, box.maxLon),
+ LatLonPointDistanceComparator.haversin1(latitude, longitude, box.maxLat, longitude));
+ assert LatLonPointDistanceComparator.haversin2(minPartialDistance) >= radiusMeters;
} else {
- box1 = box;
- box2 = null;
+ minPartialDistance = Double.POSITIVE_INFINITY;
}
return new ConstantScoreWeight(this) {
@@ -128,6 +155,22 @@ final class LatLonPointDistanceQuery extends Query {
@Override
public void visit(int docID, byte[] packedValue) {
+ // we bounds check individual values, as subtrees may cross, but we are being sent the values anyway:
+ // this reduces the amount of docvalues fetches (improves approximation)
+
+ if (StringHelper.compare(Integer.BYTES, packedValue, 0, maxLat, 0) > 0 ||
+ StringHelper.compare(Integer.BYTES, packedValue, 0, minLat, 0) < 0) {
+ // latitude out of bounding box range
+ return;
+ }
+
+ if ((StringHelper.compare(Integer.BYTES, packedValue, Integer.BYTES, maxLon, 0) > 0 ||
+ StringHelper.compare(Integer.BYTES, packedValue, Integer.BYTES, minLon, 0) < 0)
+ && StringHelper.compare(Integer.BYTES, packedValue, Integer.BYTES, minLon2, 0) < 0) {
+ // longitude out of bounding box range
+ return;
+ }
+
result.add(docID);
}
@@ -137,18 +180,25 @@ final class LatLonPointDistanceQuery extends Query {
// 3. recurse naively.
@Override
public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- double latMin = LatLonPoint.decodeLatitude(minPackedValue, 0);
- double lonMin = LatLonPoint.decodeLongitude(minPackedValue, Integer.BYTES);
- double latMax = LatLonPoint.decodeLatitude(maxPackedValue, 0);
- double lonMax = LatLonPoint.decodeLongitude(maxPackedValue, Integer.BYTES);
-
- if (latMax < box1.minLat || latMin > box1.maxLat) {
+ if (StringHelper.compare(Integer.BYTES, minPackedValue, 0, maxLat, 0) > 0 ||
+ StringHelper.compare(Integer.BYTES, maxPackedValue, 0, minLat, 0) < 0) {
// latitude out of bounding box range
return Relation.CELL_OUTSIDE_QUERY;
- } else if ((lonMax < box1.minLon || lonMin > box1.maxLon) && (box2 == null || lonMax < box2.minLon)) {
+ }
+
+ if ((StringHelper.compare(Integer.BYTES, minPackedValue, Integer.BYTES, maxLon, 0) > 0 ||
+ StringHelper.compare(Integer.BYTES, maxPackedValue, Integer.BYTES, minLon, 0) < 0)
+ && StringHelper.compare(Integer.BYTES, maxPackedValue, Integer.BYTES, minLon2, 0) < 0) {
// longitude out of bounding box range
return Relation.CELL_OUTSIDE_QUERY;
- } else if (lonMax - longitude < 90 && longitude - lonMin < 90 &&
+ }
+
+ double latMin = LatLonPoint.decodeLatitude(minPackedValue, 0);
+ double lonMin = LatLonPoint.decodeLongitude(minPackedValue, Integer.BYTES);
+ double latMax = LatLonPoint.decodeLatitude(maxPackedValue, 0);
+ double lonMax = LatLonPoint.decodeLongitude(maxPackedValue, Integer.BYTES);
+
+ if (lonMax - longitude < 90 && longitude - lonMin < 90 &&
GeoDistanceUtils.haversin(latitude, longitude, latMin, lonMin) <= radiusMeters &&
GeoDistanceUtils.haversin(latitude, longitude, latMin, lonMax) <= radiusMeters &&
GeoDistanceUtils.haversin(latitude, longitude, latMax, lonMin) <= radiusMeters &&
@@ -183,7 +233,15 @@ final class LatLonPointDistanceQuery extends Query {
long encoded = docValues.valueAt(i);
double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32));
double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF));
- if (GeoDistanceUtils.haversin(latitude, longitude, docLatitude, docLongitude) <= radiusMeters) {
+
+ // first check the partial distance, if its more than that, it can't be <= radiusMeters
+ double h1 = LatLonPointDistanceComparator.haversin1(latitude, longitude, docLatitude, docLongitude);
+ if (h1 > minPartialDistance) {
+ continue;
+ }
+
+ // fully confirm with part 2:
+ if (LatLonPointDistanceComparator.haversin2(h1) <= radiusMeters) {
return true;
}
}
[39/50] lucene-solr:jira/SOLR-445: SOLR-8838: Remove obsolete comment
Posted by ho...@apache.org.
SOLR-8838: Remove obsolete comment
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3cdde08f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3cdde08f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3cdde08f
Branch: refs/heads/jira/SOLR-445
Commit: 3cdde08ff2b77c52aa2eeb12c936da8c118e6dc2
Parents: 6e55135
Author: Steve Rowe <sa...@apache.org>
Authored: Wed Mar 16 18:58:41 2016 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Wed Mar 16 18:58:41 2016 -0400
----------------------------------------------------------------------
.../src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3cdde08f/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java
index ac245cd..73000b0 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java
@@ -71,7 +71,6 @@ public class TestUseDocValuesAsStored extends AbstractBadConfigTestBase {
private static final Pattern STORED_FIELD_NAME_PATTERN = Pattern.compile("_dv$");
static {
- // Copy of DateTimeFormatter.ISO_INSTANT with fixed 3 digit milliseconds
START_RANDOM_EPOCH_MILLIS = LocalDateTime.of(1970, Month.JANUARY, 1, 0, 0)
.toInstant(ZoneOffset.UTC).toEpochMilli();
END_RANDOM_EPOCH_MILLIS = LocalDateTime.of(2030, Month.DECEMBER, 31, 23, 59, 59, 999_000_000)
[21/50] lucene-solr:jira/SOLR-445: LUCENE-7104: remove "sort missing
first" from LatLonPoint.newDistanceSort and simplify/speedup code
Posted by ho...@apache.org.
LUCENE-7104: remove "sort missing first" from LatLonPoint.newDistanceSort and simplify/speedup code
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/02bb6c01
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/02bb6c01
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/02bb6c01
Branch: refs/heads/jira/SOLR-445
Commit: 02bb6c01550771fb0d7a75d4b283c897024c923b
Parents: 1660b56
Author: Robert Muir <rm...@apache.org>
Authored: Mon Mar 14 19:07:30 2016 -0400
Committer: Robert Muir <rm...@apache.org>
Committed: Mon Mar 14 19:07:30 2016 -0400
----------------------------------------------------------------------
.../org/apache/lucene/document/LatLonPoint.java | 3 +-
.../document/LatLonPointDistanceComparator.java | 116 +++++++------------
.../document/LatLonPointDistanceQuery.java | 8 +-
.../lucene/document/LatLonPointSortField.java | 14 +--
.../document/TestLatLonPointDistanceSort.java | 57 +++------
5 files changed, 69 insertions(+), 129 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/02bb6c01/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
index 9677baa..f5541bd 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
@@ -333,8 +333,7 @@ public class LatLonPoint extends Field {
* the hits contains a Double instance with the distance in meters.
* <p>
* If a document is missing the field, then by default it is treated as having {@link Double#POSITIVE_INFINITY} distance
- * (missing values sort last). You can change this to sort missing values first by calling
- * {@link SortField#setMissingValue(Object) setMissingValue(Double.NEGATIVE_INFINITY)} on the returned SortField.
+ * (missing values sort last).
* <p>
* If a document contains multiple values for the field, the <i>closest</i> distance to the location is used.
* <p>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/02bb6c01/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
index ef4c3f3..2102d81 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
@@ -42,7 +42,6 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
final String field;
final double latitude;
final double longitude;
- final double missingValue;
final double[] values;
double bottom;
@@ -52,27 +51,22 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
// current bounding box(es) for the bottom distance on the PQ.
// these are pre-encoded with LatLonPoint's encoding and
// used to exclude uncompetitive hits faster.
- int minLon;
- int maxLon;
- int minLat;
- int maxLat;
-
- // crossesDateLine is true, then we have a second box to check
- boolean crossesDateLine;
- int minLon2;
- int maxLon2;
- int minLat2;
- int maxLat2;
+ int minLon = Integer.MIN_VALUE;
+ int maxLon = Integer.MAX_VALUE;
+ int minLat = Integer.MIN_VALUE;
+ int maxLat = Integer.MAX_VALUE;
+
+ // second set of longitude ranges to check (for cross-dateline case)
+ int minLon2 = Integer.MAX_VALUE;
// the number of times setBottom has been called (adversary protection)
int setBottomCounter = 0;
- public LatLonPointDistanceComparator(String field, double latitude, double longitude, int numHits, double missingValue) {
+ public LatLonPointDistanceComparator(String field, double latitude, double longitude, int numHits) {
this.field = field;
this.latitude = latitude;
this.longitude = longitude;
this.values = new double[numHits];
- this.missingValue = missingValue;
}
@Override
@@ -90,53 +84,22 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
// sampling if we get called way too much: don't make gobs of bounding
// boxes if comparator hits a worst case order (e.g. backwards distance order)
if (setBottomCounter < 1024 || (setBottomCounter & 0x3F) == 0x3F) {
- // don't pass infinite values to circleToBBox: just make a complete box.
- if (bottom == missingValue) {
- minLat = minLon = Integer.MIN_VALUE;
- maxLat = maxLon = Integer.MAX_VALUE;
- crossesDateLine = false;
+ GeoRect box = GeoUtils.circleToBBox(longitude, latitude, haversin2(bottom));
+ // pre-encode our box to our integer encoding, so we don't have to decode
+ // to double values for uncompetitive hits. This has some cost!
+ minLat = LatLonPoint.encodeLatitude(box.minLat);
+ maxLat = LatLonPoint.encodeLatitude(box.maxLat);
+ if (box.crossesDateline()) {
+ // box1
+ minLon = Integer.MIN_VALUE;
+ maxLon = LatLonPoint.encodeLongitude(box.maxLon);
+ // box2
+ minLon2 = LatLonPoint.encodeLongitude(box.minLon);
} else {
- assert Double.isFinite(bottom);
- GeoRect box = GeoUtils.circleToBBox(longitude, latitude, haversin2(bottom));
- // pre-encode our box to our integer encoding, so we don't have to decode
- // to double values for uncompetitive hits. This has some cost!
- int minLatEncoded = LatLonPoint.encodeLatitude(box.minLat);
- int maxLatEncoded = LatLonPoint.encodeLatitude(box.maxLat);
- int minLonEncoded = LatLonPoint.encodeLongitude(box.minLon);
- int maxLonEncoded = LatLonPoint.encodeLongitude(box.maxLon);
- // be sure to not introduce quantization error in our optimization, just
- // round up our encoded box safely in all directions.
- if (minLatEncoded != Integer.MIN_VALUE) {
- minLatEncoded--;
- }
- if (minLonEncoded != Integer.MIN_VALUE) {
- minLonEncoded--;
- }
- if (maxLatEncoded != Integer.MAX_VALUE) {
- maxLatEncoded++;
- }
- if (maxLonEncoded != Integer.MAX_VALUE) {
- maxLonEncoded++;
- }
- crossesDateLine = box.crossesDateline();
- // crosses dateline: split
- if (crossesDateLine) {
- // box1
- minLon = Integer.MIN_VALUE;
- maxLon = maxLonEncoded;
- minLat = minLatEncoded;
- maxLat = maxLatEncoded;
- // box2
- minLon2 = minLonEncoded;
- maxLon2 = Integer.MAX_VALUE;
- minLat2 = minLatEncoded;
- maxLat2 = maxLatEncoded;
- } else {
- minLon = minLonEncoded;
- maxLon = maxLonEncoded;
- minLat = minLatEncoded;
- maxLat = maxLatEncoded;
- }
+ minLon = LatLonPoint.encodeLongitude(box.minLon);
+ maxLon = LatLonPoint.encodeLongitude(box.maxLon);
+ // disable box2
+ minLon2 = Integer.MAX_VALUE;
}
}
setBottomCounter++;
@@ -153,24 +116,33 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
int numValues = currentDocs.count();
if (numValues == 0) {
- return Double.compare(bottom, missingValue);
+ return Double.compare(bottom, Double.POSITIVE_INFINITY);
}
- double minValue = Double.POSITIVE_INFINITY;
+ int cmp = -1;
for (int i = 0; i < numValues; i++) {
long encoded = currentDocs.valueAt(i);
+
+ // test bounding box
int latitudeBits = (int)(encoded >> 32);
+ if (latitudeBits < minLat || latitudeBits > maxLat) {
+ continue;
+ }
int longitudeBits = (int)(encoded & 0xFFFFFFFF);
- boolean outsideBox = ((latitudeBits < minLat || longitudeBits < minLon || latitudeBits > maxLat || longitudeBits > maxLon) &&
- (crossesDateLine == false || latitudeBits < minLat2 || longitudeBits < minLon2 || latitudeBits > maxLat2 || longitudeBits > maxLon2));
+ if ((longitudeBits < minLon || longitudeBits > maxLon) && (longitudeBits < minLon2)) {
+ continue;
+ }
+
// only compute actual distance if its inside "competitive bounding box"
- if (outsideBox == false) {
- double docLatitude = LatLonPoint.decodeLatitude(latitudeBits);
- double docLongitude = LatLonPoint.decodeLongitude(longitudeBits);
- minValue = Math.min(minValue, haversin1(latitude, longitude, docLatitude, docLongitude));
+ double docLatitude = LatLonPoint.decodeLatitude(latitudeBits);
+ double docLongitude = LatLonPoint.decodeLongitude(longitudeBits);
+ cmp = Math.max(cmp, Double.compare(bottom, haversin1(latitude, longitude, docLatitude, docLongitude)));
+ // once we compete in the PQ, no need to continue.
+ if (cmp > 0) {
+ return cmp;
}
}
- return Double.compare(bottom, minValue);
+ return cmp;
}
@Override
@@ -204,12 +176,8 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
double sortKey(int doc) {
currentDocs.setDocument(doc);
- int numValues = currentDocs.count();
- if (numValues == 0) {
- return missingValue;
- }
-
double minValue = Double.POSITIVE_INFINITY;
+ int numValues = currentDocs.count();
for (int i = 0; i < numValues; i++) {
long encoded = currentDocs.valueAt(i);
double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/02bb6c01/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
index 9d23986..3f86f1e 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
@@ -142,9 +142,11 @@ final class LatLonPointDistanceQuery extends Query {
double latMax = LatLonPoint.decodeLatitude(maxPackedValue, 0);
double lonMax = LatLonPoint.decodeLongitude(maxPackedValue, Integer.BYTES);
- if ((latMax < box1.minLat || lonMax < box1.minLon || latMin > box1.maxLat || lonMin > box1.maxLon) &&
- (box2 == null || latMax < box2.minLat || lonMax < box2.minLon || latMin > box2.maxLat || lonMin > box2.maxLon)) {
- // we are fully outside of bounding box(es), don't proceed any further.
+ if (latMax < box1.minLat || latMin > box1.maxLat) {
+ // latitude out of bounding box range
+ return Relation.CELL_OUTSIDE_QUERY;
+ } else if ((lonMax < box1.minLon || lonMin > box1.maxLon) && (box2 == null || lonMax < box2.minLon)) {
+ // longitude out of bounding box range
return Relation.CELL_OUTSIDE_QUERY;
} else if (lonMax - longitude < 90 && longitude - lonMin < 90 &&
GeoDistanceUtils.haversin(latitude, longitude, latMin, lonMin) <= radiusMeters &&
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/02bb6c01/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java
index da90b86..f883043 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java
@@ -47,7 +47,7 @@ final class LatLonPointSortField extends SortField {
@Override
public FieldComparator<?> getComparator(int numHits, int sortPos) throws IOException {
- return new LatLonPointDistanceComparator(getField(), latitude, longitude, numHits, getMissingValue());
+ return new LatLonPointDistanceComparator(getField(), latitude, longitude, numHits);
}
@Override
@@ -57,16 +57,10 @@ final class LatLonPointSortField extends SortField {
@Override
public void setMissingValue(Object missingValue) {
- if (missingValue == null) {
- throw new IllegalArgumentException("Missing value cannot be null");
+ if (Double.valueOf(Double.POSITIVE_INFINITY).equals(missingValue) == false) {
+ throw new IllegalArgumentException("Missing value can only be Double.POSITIVE_INFINITY (missing values last), but got " + missingValue);
}
- if (missingValue.getClass() != Double.class)
- throw new IllegalArgumentException("Missing value can only be of type java.lang.Double, but got " + missingValue.getClass());
- Double value = (Double) missingValue;
- if (!Double.isInfinite(value)) {
- throw new IllegalArgumentException("Missing value can only be Double.NEGATIVE_INFINITY (missing values first) or Double.POSITIVE_INFINITY (missing values last), but got " + value);
- }
- this.missingValue = value;
+ this.missingValue = missingValue;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/02bb6c01/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
index 7df956f..4376313 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
@@ -28,6 +28,8 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.spatial.util.GeoDistanceUtils;
+import org.apache.lucene.spatial.util.GeoRect;
+import org.apache.lucene.spatial.util.GeoUtils;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -110,45 +112,6 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
dir.close();
}
- /** Add two points (one doc missing) and sort by distance */
- public void testMissingFirst() throws Exception {
- Directory dir = newDirectory();
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
-
- // missing
- Document doc = new Document();
- iw.addDocument(doc);
-
- doc = new Document();
- doc.add(new LatLonPoint("location", 40.718266, -74.007819));
- iw.addDocument(doc);
-
- doc = new Document();
- doc.add(new LatLonPoint("location", 40.7051157, -74.0088305));
- iw.addDocument(doc);
-
- IndexReader reader = iw.getReader();
- IndexSearcher searcher = newSearcher(reader);
- iw.close();
-
- SortField sortField = LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731);
- sortField.setMissingValue(Double.NEGATIVE_INFINITY);
- Sort sort = new Sort(sortField);
- TopDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort);
-
- FieldDoc d = (FieldDoc) td.scoreDocs[0];
- assertEquals(Double.NEGATIVE_INFINITY, (Double)d.fields[0], 0.0D);
-
- d = (FieldDoc) td.scoreDocs[1];
- assertEquals(462.61748421408186D, (Double)d.fields[0], 0.0D);
-
- d = (FieldDoc) td.scoreDocs[2];
- assertEquals(1056.1630445911035D, (Double)d.fields[0], 0.0D);
-
- reader.close();
- dir.close();
- }
-
/** Run a few iterations with just 10 docs, hopefully easy to debug */
public void testRandom() throws Exception {
for (int iters = 0; iters < 100; iters++) {
@@ -239,7 +202,7 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
for (int i = 0; i < numQueries; i++) {
double lat = -90 + 180.0 * random().nextDouble();
double lon = -180 + 360.0 * random().nextDouble();
- double missingValue = random().nextBoolean() ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
+ double missingValue = Double.POSITIVE_INFINITY;
Result expected[] = new Result[reader.maxDoc()];
@@ -309,4 +272,18 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
assertEquals(expected, actual);
}
}
+
+ /** Test infinite radius covers whole earth */
+ public void testInfiniteRect() {
+ for (int i = 0; i < 100000; i++) {
+ double centerLat = -90 + 180.0 * random().nextDouble();
+ double centerLon = -180 + 360.0 * random().nextDouble();
+ GeoRect rect = GeoUtils.circleToBBox(centerLat, centerLon, Double.POSITIVE_INFINITY);
+ assertEquals(-180.0, rect.minLon, 0.0D);
+ assertEquals(180.0, rect.maxLon, 0.0D);
+ assertEquals(-90.0, rect.minLat, 0.0D);
+ assertEquals(90.0, rect.maxLat, 0.0D);
+ assertFalse(rect.crossesDateline());
+ }
+ }
}
[37/50] lucene-solr:jira/SOLR-445: SOLR-8740: use docValues for
non-text fields in schema templates
Posted by ho...@apache.org.
SOLR-8740: use docValues for non-text fields in schema templates
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e76fa568
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e76fa568
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e76fa568
Branch: refs/heads/jira/SOLR-445
Commit: e76fa568172173feeed3eaaf7de06b773b32605d
Parents: 022877f
Author: yonik <yo...@apache.org>
Authored: Wed Mar 16 18:51:50 2016 -0400
Committer: yonik <yo...@apache.org>
Committed: Wed Mar 16 18:51:50 2016 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 9 ++++
.../basic_configs/conf/managed-schema | 32 ++++++------
.../conf/managed-schema | 53 ++++++++++----------
.../conf/managed-schema | 43 ++++++----------
4 files changed, 66 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e76fa568/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index eaedca6..945a123 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -130,6 +130,12 @@ Upgrading from Solr 5.x
* SOLR-8736: The deprecated GET methods for schema are now accessible through the bulk API. The output
has less details and is not backward compatible.
+* In the past, Solr guaranteed that retrieval of multi-valued fields would preserve the order of values.
+ Because values may now be retrieved from column-stored fields (docValues="true"), in conjunction with the
+ fact that docValues do not currently preserve order, means that users should set useDocValuesAsStored="false"
+ to prevent future optizations from using the column-stored values over the row-stored values when
+ fields have both stored="true" and docValues="true".
+
Detailed Change List
----------------------
@@ -442,6 +448,9 @@ Other Changes
* SOLR-8836: Return 400, and a SolrException when an invalid json is provided to the update handler
instead of 500. (Jason Gerlowski via Anshum Gupta)
+* SOLR-8740: docValues are now enabled by default for most non-text (string, date, and numeric) fields
+ in the schema templates. (yonik)
+
================== 5.5.1 ==================
Bug Fixes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e76fa568/solr/server/solr/configsets/basic_configs/conf/managed-schema
----------------------------------------------------------------------
diff --git a/solr/server/solr/configsets/basic_configs/conf/managed-schema b/solr/server/solr/configsets/basic_configs/conf/managed-schema
index 2599a28..3ce6b73 100644
--- a/solr/server/solr/configsets/basic_configs/conf/managed-schema
+++ b/solr/server/solr/configsets/basic_configs/conf/managed-schema
@@ -94,12 +94,12 @@
<!-- If you remove this field, you must _also_ disable the update log in solrconfig.xml
or Solr won't start. _version_ and update log are required for SolrCloud
-->
- <field name="_version_" type="long" indexed="true" stored="true"/>
+ <field name="_version_" type="long" indexed="true" stored="false" />
<!-- points to the root document of a block of nested documents. Required for nested
document support, may be removed otherwise
-->
- <field name="_root_" type="string" indexed="true" stored="false"/>
+ <field name="_root_" type="string" indexed="true" stored="false" docValues="false" />
<!-- Only remove the "id" field if you have a very good reason to. While not strictly
required, it is highly recommended. A <uniqueKey> is present in almost all Solr
@@ -135,7 +135,7 @@
<dynamicField name="*_ds" type="double" indexed="true" stored="true" multiValued="true"/>
<!-- Type used to index the lat and lon components for the "location" FieldType -->
- <dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false" />
+ <dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false" useDocValuesAsStored="false" />
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
<dynamicField name="*_dts" type="date" indexed="true" stored="true" multiValued="true"/>
@@ -187,7 +187,7 @@
It supports doc values but in that case the field needs to be
single-valued and either required or have a default value.
-->
- <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" docValues="true" />
<!-- boolean type: "true" or "false" -->
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
@@ -214,10 +214,10 @@
These fields support doc values, but they require the field to be
single-valued and either be required or have a default value.
-->
- <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="int" class="solr.TrieIntField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<!--
Numeric field types that index each value at various levels of precision
@@ -229,10 +229,10 @@
indexed per value, slightly larger index size, and faster range queries.
A precisionStep of 0 disables indexing at different precision levels.
-->
- <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tint" class="solr.TrieIntField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="solr.TrieLongField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
<!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
is a more restricted form of the canonical representation of dateTime
@@ -256,10 +256,10 @@
Note: For faster range queries, consider the tdate type
-->
- <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="date" class="solr.TrieDateField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<!-- A Trie based date field for faster date range queries and date faceting. -->
- <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
+ <fieldType name="tdate" class="solr.TrieDateField" docValues="true" precisionStep="6" positionIncrementGap="0"/>
<!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
@@ -487,7 +487,7 @@
<!-- since fields of this type are by default not stored or indexed,
any data added to them will be ignored outright. -->
- <fieldType name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />
+ <fieldType name="ignored" stored="false" indexed="false" docValues="false" multiValued="true" class="solr.StrField" />
<!-- This point type indexes the coordinates as separate fields (subFields)
If subFieldType is defined, it references a type, and a dynamic field
@@ -517,7 +517,7 @@
relevancy. -->
<fieldType name="bbox" class="solr.BBoxField"
geo="true" distanceUnits="kilometers" numberType="_bbox_coord" />
- <fieldType name="_bbox_coord" class="solr.TrieDoubleField" precisionStep="8" docValues="true" stored="false"/>
+ <fieldType name="_bbox_coord" class="solr.TrieDoubleField" precisionStep="8" docValues="true" useDocValuesAsStored="false" stored="false"/>
<!-- Money/currency field type. See http://wiki.apache.org/solr/MoneyFieldType
Parameters:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e76fa568/solr/server/solr/configsets/data_driven_schema_configs/conf/managed-schema
----------------------------------------------------------------------
diff --git a/solr/server/solr/configsets/data_driven_schema_configs/conf/managed-schema b/solr/server/solr/configsets/data_driven_schema_configs/conf/managed-schema
index 822e207..fa9429f 100644
--- a/solr/server/solr/configsets/data_driven_schema_configs/conf/managed-schema
+++ b/solr/server/solr/configsets/data_driven_schema_configs/conf/managed-schema
@@ -118,8 +118,8 @@
If you don't need it, consider removing it and the corresponding copyField directive.
-->
<field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
- <field name="_version_" type="long" indexed="true" stored="true"/>
- <field name="_root_" type="string" indexed="true" stored="false"/>
+ <field name="_version_" type="long" indexed="true" stored="false"/>
+ <field name="_root_" type="string" indexed="true" stored="false" docValues="false" />
<field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/>
<copyField source="*" dest="_text_"/>
@@ -146,7 +146,7 @@
<dynamicField name="*_ds" type="doubles" indexed="true" stored="true"/>
<!-- Type used to index the lat and lon components for the "location" FieldType -->
- <dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false" />
+ <dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false" useDocValuesAsStored="false" />
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
<dynamicField name="*_dts" type="date" indexed="true" stored="true" multiValued="true"/>
@@ -205,8 +205,8 @@
It supports doc values but in that case the field needs to be
single-valued and either required or have a default value.
-->
- <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
- <fieldType name="strings" class="solr.StrField" sortMissingLast="true" multiValued="true"/>
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" docValues="true" />
+ <fieldType name="strings" class="solr.StrField" sortMissingLast="true" multiValued="true" docValues="true" />
<!-- boolean type: "true" or "false" -->
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
@@ -235,15 +235,15 @@
These fields support doc values, but they require the field to be
single-valued and either be required or have a default value.
-->
- <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="int" class="solr.TrieIntField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="ints" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0" multiValued="true"/>
- <fieldType name="floats" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0" multiValued="true"/>
- <fieldType name="longs" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0" multiValued="true"/>
- <fieldType name="doubles" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0" multiValued="true"/>
+ <fieldType name="ints" class="solr.TrieIntField" docValues="true" precisionStep="0" positionIncrementGap="0" multiValued="true"/>
+ <fieldType name="floats" class="solr.TrieFloatField" docValues="true" precisionStep="0" positionIncrementGap="0" multiValued="true"/>
+ <fieldType name="longs" class="solr.TrieLongField" docValues="true" precisionStep="0" positionIncrementGap="0" multiValued="true"/>
+ <fieldType name="doubles" class="solr.TrieDoubleField" docValues="true" precisionStep="0" positionIncrementGap="0" multiValued="true"/>
<!--
Numeric field types that index each value at various levels of precision
@@ -255,15 +255,15 @@
indexed per value, slightly larger index size, and faster range queries.
A precisionStep of 0 disables indexing at different precision levels.
-->
- <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tint" class="solr.TrieIntField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="solr.TrieLongField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tints" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
- <fieldType name="tfloats" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
- <fieldType name="tlongs" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
- <fieldType name="tdoubles" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
+ <fieldType name="tints" class="solr.TrieIntField" docValues="true" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
+ <fieldType name="tfloats" class="solr.TrieFloatField" docValues="true" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
+ <fieldType name="tlongs" class="solr.TrieLongField" docValues="true" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
+ <fieldType name="tdoubles" class="solr.TrieDoubleField" docValues="true" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
<!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
is a more restricted form of the canonical representation of dateTime
@@ -287,13 +287,13 @@
Note: For faster range queries, consider the tdate type
-->
- <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="dates" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0" multiValued="true"/>
+ <fieldType name="date" class="solr.TrieDateField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="dates" class="solr.TrieDateField" docValues="true" precisionStep="0" positionIncrementGap="0" multiValued="true"/>
<!-- A Trie based date field for faster date range queries and date faceting. -->
- <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
+ <fieldType name="tdate" class="solr.TrieDateField" docValues="true" precisionStep="6" positionIncrementGap="0"/>
- <fieldType name="tdates" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0" multiValued="true"/>
+ <fieldType name="tdates" class="solr.TrieDateField" docValues="true" precisionStep="6" positionIncrementGap="0" multiValued="true"/>
<!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
@@ -532,7 +532,7 @@
<!-- since fields of this type are by default not stored or indexed,
any data added to them will be ignored outright. -->
- <fieldType name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />
+ <fieldType name="ignored" stored="false" indexed="false" docValues="false" multiValued="true" class="solr.StrField" />
<!-- This point type indexes the coordinates as separate fields (subFields)
If subFieldType is defined, it references a type, and a dynamic field
@@ -1000,5 +1000,4 @@
</similarity>
-->
-<useDocValuesAsStored>false</useDocValuesAsStored>
</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e76fa568/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema
----------------------------------------------------------------------
diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema b/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema
index ddba483..1000d9a 100644
--- a/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema
+++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema
@@ -112,12 +112,12 @@
<!-- If you remove this field, you must _also_ disable the update log in solrconfig.xml
or Solr won't start. _version_ and update log are required for SolrCloud
-->
- <field name="_version_" type="long" indexed="true" stored="true"/>
+ <field name="_version_" type="long" indexed="true" stored="false" />
<!-- points to the root document of a block of nested documents. Required for nested
document support, may be removed otherwise
-->
- <field name="_root_" type="string" indexed="true" stored="false"/>
+ <field name="_root_" type="string" indexed="true" stored="false" docValues="false" />
<!-- Only remove the "id" field if you have a very good reason to. While not strictly
required, it is highly recommended. A <uniqueKey> is present in almost all Solr
@@ -184,24 +184,11 @@
<!-- non-tokenized version of manufacturer to make it easier to sort or group
results by manufacturer. copied from "manu" via copyField -->
- <field name="manu_exact" type="string" indexed="true" stored="false"/>
+ <field name="manu_exact" type="string" indexed="true" stored="false" docValues="false" />
<field name="payloads" type="payloads" indexed="true" stored="true"/>
- <!--
- Some fields such as popularity and manu_exact could be modified to
- leverage doc values:
- <field name="popularity" type="int" indexed="true" stored="true" docValues="true" />
- <field name="manu_exact" type="string" indexed="false" stored="false" docValues="true" />
- <field name="cat" type="string" indexed="true" stored="true" docValues="true" multiValued="true"/>
-
-
- Although it would make indexing slightly slower and the index bigger, it
- would also make the index faster to load, more memory-efficient and more
- NRT-friendly.
- -->
-
<!-- Dynamic field definitions allow using convention over configuration
for fields via the specification of patterns to match field names.
EXAMPLE: name="*_i" will match any field ending in _i (like myid_i, z_i)
@@ -225,7 +212,7 @@
<dynamicField name="*_ds" type="double" indexed="true" stored="true" multiValued="true"/>
<!-- Type used to index the lat and lon components for the "location" FieldType -->
- <dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false" />
+ <dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false" useDocValuesAsStored="false" />
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
<dynamicField name="*_dts" type="date" indexed="true" stored="true" multiValued="true"/>
@@ -351,10 +338,10 @@
These fields support doc values, but they require the field to be
single-valued and either be required or have a default value.
-->
- <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="int" class="solr.TrieIntField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<!--
Numeric field types that index each value at various levels of precision
@@ -366,10 +353,10 @@
indexed per value, slightly larger index size, and faster range queries.
A precisionStep of 0 disables indexing at different precision levels.
-->
- <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tint" class="solr.TrieIntField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="solr.TrieLongField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
<!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
is a more restricted form of the canonical representation of dateTime
@@ -393,10 +380,10 @@
Note: For faster range queries, consider the tdate type
-->
- <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="date" class="solr.TrieDateField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<!-- A Trie based date field for faster date range queries and date faceting. -->
- <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
+ <fieldType name="tdate" class="solr.TrieDateField" docValues="true" precisionStep="6" positionIncrementGap="0"/>
<!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
@@ -723,7 +710,7 @@
relevancy. -->
<fieldType name="bbox" class="solr.BBoxField"
geo="true" distanceUnits="kilometers" numberType="_bbox_coord" />
- <fieldType name="_bbox_coord" class="solr.TrieDoubleField" precisionStep="8" docValues="true" stored="false"/>
+ <fieldType name="_bbox_coord" class="solr.TrieDoubleField" precisionStep="8" docValues="true" useDocValuesAsStored="false" stored="false" />
<!-- Money/currency field type. See http://wiki.apache.org/solr/MoneyFieldType
Parameters:
[13/50] lucene-solr:jira/SOLR-445: fix wrong param order in 2B tests
Posted by ho...@apache.org.
fix wrong param order in 2B tests
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3c7e55da
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3c7e55da
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3c7e55da
Branch: refs/heads/jira/SOLR-445
Commit: 3c7e55da3a29224a90a8fc71815a7a52433a6a90
Parents: 983908c
Author: Mike McCandless <mi...@apache.org>
Authored: Sun Mar 13 08:56:51 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Sun Mar 13 08:56:51 2016 -0400
----------------------------------------------------------------------
lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java | 6 +++---
.../src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java | 4 ++--
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c7e55da/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
index 75f2bbe..5766b91 100644
--- a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
+++ b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
@@ -86,7 +86,7 @@ public class Test2BPoints extends LuceneTestCase {
w.forceMerge(1);
DirectoryReader r = DirectoryReader.open(w);
IndexSearcher s = new IndexSearcher(r);
- assertEquals(1250, s.count(LongPoint.newRangeQuery("long", 33640828, 33673327)));
+ assertEquals(numDocs, s.count(LongPoint.newRangeQuery("long", Long.MIN_VALUE, Long.MAX_VALUE)));
assertTrue(r.leaves().get(0).reader().getPointValues().size("long") > Integer.MAX_VALUE);
r.close();
w.close();
@@ -117,7 +117,7 @@ public class Test2BPoints extends LuceneTestCase {
}
final int numDocs = (Integer.MAX_VALUE / 26) + 1;
- long counter = 0;
+ int counter = 0;
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
for (int j=0;j<26;j++) {
@@ -134,7 +134,7 @@ public class Test2BPoints extends LuceneTestCase {
w.forceMerge(1);
DirectoryReader r = DirectoryReader.open(w);
IndexSearcher s = new IndexSearcher(r);
- assertEquals(1250, s.count(LongPoint.newRangeQuery("long", new long[] {33640828, 33673327}, new long[] {Long.MIN_VALUE, Long.MAX_VALUE})));
+ assertEquals(numDocs, s.count(LongPoint.newRangeQuery("long", new long[] {Long.MIN_VALUE, Long.MAX_VALUE}, new long[] {Long.MIN_VALUE, Long.MAX_VALUE})));
assertTrue(r.leaves().get(0).reader().getPointValues().size("long") > Integer.MAX_VALUE);
r.close();
w.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c7e55da/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
index eb3aa47..4df2ebe 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
@@ -55,7 +55,7 @@ public class Test2BBKDPoints extends LuceneTestCase {
final int numDocs = (Integer.MAX_VALUE / 26) + 100;
- BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, 1024, 256, Long.BYTES, 26L * numDocs);
+ BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, Long.BYTES, 1024, 256, 26L * numDocs);
int counter = 0;
byte[] packedBytes = new byte[Long.BYTES];
for (int docID = 0; docID < numDocs; docID++) {
@@ -88,7 +88,7 @@ public class Test2BBKDPoints extends LuceneTestCase {
final int numDocs = (Integer.MAX_VALUE / 26) + 100;
- BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, 1024, 256, Long.BYTES, 26L * numDocs);
+ BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES, 1024, 256, 26L * numDocs);
int counter = 0;
byte[] packedBytes = new byte[2*Long.BYTES];
for (int docID = 0; docID < numDocs; docID++) {
[27/50] lucene-solr:jira/SOLR-445: LUCENE-7093: Added point values
support to the memory index
Posted by ho...@apache.org.
LUCENE-7093: Added point values support to the memory index
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c1dfeb8e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c1dfeb8e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c1dfeb8e
Branch: refs/heads/jira/SOLR-445
Commit: c1dfeb8ef85be924f17f8aece46d008382d538e9
Parents: 82c0619
Author: Martijn van Groningen <mv...@apache.org>
Authored: Tue Mar 15 12:32:46 2016 +0100
Committer: Martijn van Groningen <mv...@apache.org>
Committed: Tue Mar 15 13:13:19 2016 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../apache/lucene/index/memory/MemoryIndex.java | 185 +++++++++++++++++--
.../lucene/index/memory/TestMemoryIndex.java | 160 ++++++++++++++++
.../memory/TestMemoryIndexAgainstRAMDir.java | 58 ++++++
4 files changed, 389 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1dfeb8e/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 61737d9..59cd092 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -209,6 +209,9 @@ Other
* LUCENE-7091: Add doc values support to MemoryIndex
(Martijn van Groningen, David Smiley)
+* LUCENE-7093: Add point values support to MemoryIndex
+ (Martijn van Groningen, Mike McCandless)
+
======================= Lucene 5.5.0 =======================
New Features
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1dfeb8e/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index 40159aa..58a1017 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -258,7 +258,8 @@ public class MemoryIndex {
throw new IllegalArgumentException("analyzer must not be null");
TokenStream stream = analyzer.tokenStream(fieldName, text);
- addField(fieldName, stream, 1.0f, analyzer.getPositionIncrementGap(fieldName), analyzer.getOffsetGap(fieldName), DocValuesType.NONE, null);
+ addField(fieldName, stream, 1.0f, analyzer.getPositionIncrementGap(fieldName), analyzer.getOffsetGap(fieldName),
+ DocValuesType.NONE, null, 0, 0, null);
}
/**
@@ -377,10 +378,6 @@ public class MemoryIndex {
* structures are not supported by MemoryIndex
*/
public void addField(IndexableField field, Analyzer analyzer, float boost) {
- if (field.fieldType().pointDimensionCount() != 0) {
- throw new IllegalArgumentException("MemoryIndex does not support Points");
- }
-
int offsetGap;
TokenStream tokenStream;
int positionIncrementGap;
@@ -412,7 +409,12 @@ public class MemoryIndex {
default:
throw new UnsupportedOperationException("unknown doc values type [" + docValuesType + "]");
}
- addField(field.name(), tokenStream, boost, positionIncrementGap, offsetGap, docValuesType, docValuesValue);
+ BytesRef pointValue = null;
+ if (field.fieldType().pointDimensionCount() > 0) {
+ pointValue = field.binaryValue();
+ }
+ addField(field.name(), tokenStream, boost, positionIncrementGap, offsetGap, docValuesType, docValuesValue,
+ field.fieldType().pointDimensionCount(), field.fieldType().pointNumBytes(), pointValue);
}
/**
@@ -481,11 +483,12 @@ public class MemoryIndex {
* @see org.apache.lucene.document.Field#setBoost(float)
*/
public void addField(String fieldName, TokenStream tokenStream, float boost, int positionIncrementGap, int offsetGap) {
- addField(fieldName, tokenStream, boost, positionIncrementGap, offsetGap, DocValuesType.NONE, null);
+ addField(fieldName, tokenStream, boost, positionIncrementGap, offsetGap, DocValuesType.NONE, null, 0, 0, null);
}
private void addField(String fieldName, TokenStream tokenStream, float boost, int positionIncrementGap, int offsetGap,
- DocValuesType docValuesType, Object docValuesValue) {
+ DocValuesType docValuesType, Object docValuesValue, int pointDimensionCount, int pointNumBytes,
+ BytesRef pointValue) {
if (frozen) {
throw new IllegalArgumentException("Cannot call addField() when MemoryIndex is frozen");
@@ -503,7 +506,9 @@ public class MemoryIndex {
FieldInfo fieldInfo = new FieldInfo(fieldName, fields.size(), true, false, storePayloads, indexOptions, docValuesType, -1, Collections.emptyMap(), 0, 0);
fields.put(fieldName, info = new Info(fieldInfo, byteBlockPool));
}
-
+ if (pointDimensionCount > 0) {
+ storePointValues(info, pointDimensionCount, pointNumBytes, pointValue);
+ }
if (docValuesType != DocValuesType.NONE) {
storeDocValues(info, docValuesType, docValuesValue);
}
@@ -512,6 +517,15 @@ public class MemoryIndex {
}
}
+ private void storePointValues(Info info, int pointDimensionCount, int pointNumBytes, BytesRef pointValue) {
+ info.fieldInfo.setPointDimensions(pointDimensionCount, pointNumBytes);
+ if (info.pointValues == null) {
+ info.pointValues = new BytesRef[4];
+ }
+ info.pointValues = ArrayUtil.grow(info.pointValues, info.pointValuesCount + 1);
+ info.pointValues[info.pointValuesCount++] = BytesRef.deepCopyOf(pointValue);
+ }
+
private void storeDocValues(Info info, DocValuesType docValuesType, Object docValuesValue) {
String fieldName = info.fieldInfo.name;
DocValuesType existingDocValuesType = info.fieldInfo.getDocValuesType();
@@ -829,7 +843,15 @@ public class MemoryIndex {
private NumericDocValuesProducer numericProducer;
- private boolean preparedDocValues;
+ private boolean preparedDocValuesAndPointValues;
+
+ private BytesRef[] pointValues;
+
+ private byte[] minPackedValue;
+
+ private byte[] maxPackedValue;
+
+ private int pointValuesCount;
private Info(FieldInfo fieldInfo, ByteBlockPool byteBlockPool) {
this.fieldInfo = fieldInfo;
@@ -841,7 +863,7 @@ public class MemoryIndex {
void freeze() {
sortTerms();
- prepareDocValues();
+ prepareDocValuesAndPointValues();
getNormDocValues();
}
@@ -859,8 +881,8 @@ public class MemoryIndex {
}
}
- void prepareDocValues() {
- if (preparedDocValues == false) {
+ void prepareDocValuesAndPointValues() {
+ if (preparedDocValuesAndPointValues == false) {
DocValuesType dvType = fieldInfo.getDocValuesType();
if (dvType == DocValuesType.NUMERIC || dvType == DocValuesType.SORTED_NUMERIC) {
numericProducer.prepareForUsage();
@@ -868,7 +890,30 @@ public class MemoryIndex {
if (dvType == DocValuesType.BINARY || dvType == DocValuesType.SORTED || dvType == DocValuesType.SORTED_SET) {
binaryProducer.prepareForUsage();
}
- preparedDocValues = true;
+ if (pointValues != null) {
+ assert pointValues[0].bytes.length == pointValues[0].length : "BytesRef should wrap a precise byte[], BytesRef.deepCopyOf() should take care of this";
+
+ final int numDimensions = fieldInfo.getPointDimensionCount();
+ final int numBytesPerDimension = fieldInfo.getPointNumBytes();
+ minPackedValue = pointValues[0].bytes.clone();
+ maxPackedValue = pointValues[0].bytes.clone();
+
+ for (int i = 0; i < pointValuesCount; i++) {
+ BytesRef pointValue = pointValues[i];
+ assert pointValue.bytes.length == pointValue.length : "BytesRef should wrap a precise byte[], BytesRef.deepCopyOf() should take care of this";
+
+ for (int dim = 0; dim < numDimensions; ++dim) {
+ int offset = dim * numBytesPerDimension;
+ if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, minPackedValue, offset) < 0) {
+ System.arraycopy(pointValue.bytes, offset, minPackedValue, offset, numBytesPerDimension);
+ }
+ if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, maxPackedValue, offset) > 0) {
+ System.arraycopy(pointValue.bytes, offset, maxPackedValue, offset, numBytesPerDimension);
+ }
+ }
+ }
+ }
+ preparedDocValuesAndPointValues = true;
}
}
@@ -977,11 +1022,22 @@ public class MemoryIndex {
* required by the Lucene IndexReader contracts.
*/
private final class MemoryIndexReader extends LeafReader {
-
+
+ private final PointValues pointValues;
+
private MemoryIndexReader() {
super(); // avoid as much superclass baggage as possible
+ boolean hasPointValues = false;
for (Info info : fields.values()) {
- info.prepareDocValues();
+ info.prepareDocValuesAndPointValues();
+ if (info.pointValues != null) {
+ hasPointValues = true;
+ }
+ }
+ if (hasPointValues) {
+ pointValues = new MemoryIndexPointValues();
+ } else {
+ pointValues = null;
}
}
@@ -1111,7 +1167,7 @@ public class MemoryIndex {
@Override
public PointValues getPointValues() {
- return null;
+ return pointValues;
}
@Override
@@ -1412,6 +1468,101 @@ public class MemoryIndex {
return 1;
}
}
+
+ private class MemoryIndexPointValues extends PointValues {
+
+ @Override
+ public void intersect(String fieldName, IntersectVisitor visitor) throws IOException {
+ Info info = fields.get(fieldName);
+ if (info == null) {
+ return;
+ }
+ BytesRef[] values = info.pointValues;
+ if (values == null) {
+ return;
+ }
+
+ visitor.grow(info.pointValuesCount);
+ for (int i = 0; i < info.pointValuesCount; i++) {
+ visitor.visit(0, values[i].bytes);
+ }
+ }
+
+ @Override
+ public byte[] getMinPackedValue(String fieldName) throws IOException {
+ Info info = fields.get(fieldName);
+ if (info == null) {
+ return null;
+ }
+ BytesRef[] values = info.pointValues;
+ if (values != null) {
+ return info.minPackedValue;
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public byte[] getMaxPackedValue(String fieldName) throws IOException {
+ Info info = fields.get(fieldName);
+ if (info == null) {
+ return null;
+ }
+ BytesRef[] values = info.pointValues;
+ if (values != null) {
+ return info.maxPackedValue;
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public int getNumDimensions(String fieldName) throws IOException {
+ Info info = fields.get(fieldName);
+ if (info == null){
+ return 0;
+ }
+ return info.fieldInfo.getPointDimensionCount();
+ }
+
+ @Override
+ public int getBytesPerDimension(String fieldName) throws IOException {
+ Info info = fields.get(fieldName);
+ if (info == null){
+ return 0;
+ }
+ return info.fieldInfo.getPointNumBytes();
+ }
+
+ @Override
+ public long size(String fieldName) {
+ Info info = fields.get(fieldName);
+ if (info == null) {
+ return 0;
+ }
+ BytesRef[] values = info.pointValues;
+ if (values != null) {
+ return info.pointValuesCount;
+ } else {
+ return 0;
+ }
+ }
+
+ @Override
+ public int getDocCount(String fieldName) {
+ Info info = fields.get(fieldName);
+ if (info == null) {
+ return 0;
+ }
+ BytesRef[] values = info.pointValues;
+ if (values != null) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ }
@Override
public Fields getTermVectors(int docID) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1dfeb8e/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
index 7282e0e..1010c13 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
@@ -17,13 +17,28 @@
package org.apache.lucene.index.memory;
import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.IntStream;
+import java.util.stream.LongStream;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockPayloadAnalyzer;
import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.BinaryPoint;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FloatPoint;
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
@@ -31,9 +46,12 @@ import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
@@ -44,7 +62,9 @@ import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.util.BytesRef;
@@ -310,4 +330,144 @@ public class TestMemoryIndex extends LuceneTestCase {
assertEquals("quick brown fox", binaryDocValues.get(0).utf8ToString());
}
+ public void testPointValues() throws Exception {
+ List<Function<Long, IndexableField>> fieldFunctions = Arrays.asList(
+ (t) -> new IntPoint("number", t.intValue()),
+ (t) -> new LongPoint("number", t),
+ (t) -> new FloatPoint("number", t.floatValue()),
+ (t) -> new DoublePoint("number", t.doubleValue())
+ );
+ List<Function<Long, Query>> exactQueryFunctions = Arrays.asList(
+ (t) -> IntPoint.newExactQuery("number", t.intValue()),
+ (t) -> LongPoint.newExactQuery("number", t),
+ (t) -> FloatPoint.newExactQuery("number", t.floatValue()),
+ (t) -> DoublePoint.newExactQuery("number", t.doubleValue())
+ );
+ List<Function<long[], Query>> setQueryFunctions = Arrays.asList(
+ (t) -> IntPoint.newSetQuery("number", LongStream.of(t).mapToInt(value -> (int) value).toArray()),
+ (t) -> LongPoint.newSetQuery("number", t),
+ (t) -> FloatPoint.newSetQuery("number", Arrays.asList(LongStream.of(t).mapToObj(value -> (float) value).toArray(Float[]::new))),
+ (t) -> DoublePoint.newSetQuery("number", LongStream.of(t).mapToDouble(value -> (double) value).toArray())
+ );
+ List<BiFunction<Long, Long, Query>> rangeQueryFunctions = Arrays.asList(
+ (t, u) -> IntPoint.newRangeQuery("number", t.intValue(), u.intValue()),
+ (t, u) -> LongPoint.newRangeQuery("number", t, u),
+ (t, u) -> FloatPoint.newRangeQuery("number", t.floatValue(), u.floatValue()),
+ (t, u) -> DoublePoint.newRangeQuery("number", t.doubleValue(), u.doubleValue())
+ );
+
+ for (int i = 0; i < fieldFunctions.size(); i++) {
+ Function<Long, IndexableField> fieldFunction = fieldFunctions.get(i);
+ Function<Long, Query> exactQueryFunction = exactQueryFunctions.get(i);
+ Function<long[], Query> setQueryFunction = setQueryFunctions.get(i);
+ BiFunction<Long, Long, Query> rangeQueryFunction = rangeQueryFunctions.get(i);
+
+ Document doc = new Document();
+ for (int number = 1; number < 32; number += 2) {
+ doc.add(fieldFunction.apply((long) number));
+ }
+ MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
+ IndexSearcher indexSearcher = mi.createSearcher();
+ Query query = exactQueryFunction.apply(5L);
+ assertEquals(1, indexSearcher.count(query));
+ query = exactQueryFunction.apply(4L);
+ assertEquals(0, indexSearcher.count(query));
+
+
+ query = setQueryFunction.apply(new long[]{3L, 9L, 19L});
+ assertEquals(1, indexSearcher.count(query));
+ query = setQueryFunction.apply(new long[]{2L, 8L, 13L});
+ assertEquals(1, indexSearcher.count(query));
+ query = setQueryFunction.apply(new long[]{2L, 8L, 16L});
+ assertEquals(0, indexSearcher.count(query));
+
+ query = rangeQueryFunction.apply(2L, 16L);
+ assertEquals(1, indexSearcher.count(query));
+ query = rangeQueryFunction.apply(24L, 48L);
+ assertEquals(1, indexSearcher.count(query));
+ query = rangeQueryFunction.apply(48L, 68L);
+ assertEquals(0, indexSearcher.count(query));
+ }
+ }
+
+ public void testPointValuesDoNotAffectBoostPositionsOrOffset() throws Exception {
+ MemoryIndex mi = new MemoryIndex(true, true);
+ mi.addField(new TextField("text", "quick brown fox", Field.Store.NO), analyzer, 5f);
+ mi.addField(new BinaryPoint("text", "quick".getBytes(StandardCharsets.UTF_8)), analyzer, 5f);
+ mi.addField(new BinaryPoint("text", "brown".getBytes(StandardCharsets.UTF_8)), analyzer, 5f);
+ LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();
+ TermsEnum tenum = leafReader.terms("text").iterator();
+
+ assertEquals("brown", tenum.next().utf8ToString());
+ PostingsEnum penum = tenum.postings(null, PostingsEnum.OFFSETS);
+ assertEquals(0, penum.nextDoc());
+ assertEquals(1, penum.freq());
+ assertEquals(1, penum.nextPosition());
+ assertEquals(6, penum.startOffset());
+ assertEquals(11, penum.endOffset());
+
+ assertEquals("fox", tenum.next().utf8ToString());
+ penum = tenum.postings(penum, PostingsEnum.OFFSETS);
+ assertEquals(0, penum.nextDoc());
+ assertEquals(1, penum.freq());
+ assertEquals(2, penum.nextPosition());
+ assertEquals(12, penum.startOffset());
+ assertEquals(15, penum.endOffset());
+
+ assertEquals("quick", tenum.next().utf8ToString());
+ penum = tenum.postings(penum, PostingsEnum.OFFSETS);
+ assertEquals(0, penum.nextDoc());
+ assertEquals(1, penum.freq());
+ assertEquals(0, penum.nextPosition());
+ assertEquals(0, penum.startOffset());
+ assertEquals(5, penum.endOffset());
+
+ IndexSearcher indexSearcher = mi.createSearcher();
+ assertEquals(1, indexSearcher.count(BinaryPoint.newExactQuery("text", "quick".getBytes(StandardCharsets.UTF_8))));
+ assertEquals(1, indexSearcher.count(BinaryPoint.newExactQuery("text", "brown".getBytes(StandardCharsets.UTF_8))));
+ assertEquals(0, indexSearcher.count(BinaryPoint.newExactQuery("text", "jumps".getBytes(StandardCharsets.UTF_8))));
+ }
+
+ public void test2DPoints() throws Exception {
+ Document doc = new Document();
+ doc.add(new IntPoint("ints", 0, -100));
+ doc.add(new IntPoint("ints", 20, 20));
+ doc.add(new IntPoint("ints", 100, -100));
+ doc.add(new LongPoint("longs", 0L, -100L));
+ doc.add(new LongPoint("longs", 20L, 20L));
+ doc.add(new LongPoint("longs", 100L, -100L));
+ doc.add(new FloatPoint("floats", 0F, -100F));
+ doc.add(new FloatPoint("floats", 20F, 20F));
+ doc.add(new FloatPoint("floats", 100F, -100F));
+ doc.add(new DoublePoint("doubles", 0D, -100D));
+ doc.add(new DoublePoint("doubles", 20D, 20D));
+ doc.add(new DoublePoint("doubles", 100D, -100D));
+
+ MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
+ IndexSearcher s = mi.createSearcher();
+
+ assertEquals(1, s.count(IntPoint.newRangeQuery("ints", new int[] {10, 10}, new int[] {30, 30})));
+ assertEquals(1, s.count(LongPoint.newRangeQuery("longs", new long[] {10L, 10L}, new long[] {30L, 30L})));
+ assertEquals(1, s.count(FloatPoint.newRangeQuery("floats", new float[] {10F, 10F}, new float[] {30F, 30F})));
+ assertEquals(1, s.count(DoublePoint.newRangeQuery("doubles", new double[] {10D, 10D}, new double[] {30D, 30D})));
+ }
+
+ public void testIndexingPointsAndDocValues() throws Exception {
+ FieldType type = new FieldType();
+ type.setDimensions(1, 4);
+ type.setDocValuesType(DocValuesType.BINARY);
+ type.freeze();
+ Document doc = new Document();
+ byte[] packedPoint = "term".getBytes(StandardCharsets.UTF_8);
+ doc.add(new BinaryPoint("field", packedPoint, type));
+ MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
+ LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();
+
+ assertEquals(1, leafReader.getPointValues().size("field"));
+ assertArrayEquals(packedPoint, leafReader.getPointValues().getMinPackedValue("field"));
+ assertArrayEquals(packedPoint, leafReader.getPointValues().getMaxPackedValue("field"));
+
+ assertEquals("term", leafReader.getBinaryDocValues("field").get(0).utf8ToString());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1dfeb8e/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
index 3e6778a..ba1263e 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
@@ -21,8 +21,12 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
+import java.util.function.IntSupplier;
+import java.util.function.Supplier;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -37,9 +41,13 @@ import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FloatPoint;
+import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LegacyLongField;
+import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
@@ -70,6 +78,7 @@ import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.Query;
import org.apache.lucene.search.RegexpQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
@@ -568,6 +577,55 @@ public class TestMemoryIndexAgainstRAMDir extends BaseTokenStreamTestCase {
dir.close();
}
+ public void testPointValuesMemoryIndexVsNormalIndex() throws Exception {
+ int size = atLeast(12);
+
+ List<Integer> randomValues = new ArrayList<>();
+
+ Document doc = new Document();
+ for (Integer randomInteger : random().ints(size).toArray()) {
+ doc.add(new IntPoint("int", randomInteger));
+ randomValues.add(randomInteger);
+ doc.add(new LongPoint("long", randomInteger));
+ doc.add(new FloatPoint("float", randomInteger));
+ doc.add(new DoublePoint("double", randomInteger));
+ }
+
+ MockAnalyzer mockAnalyzer = new MockAnalyzer(random());
+ MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, mockAnalyzer);
+ IndexSearcher memoryIndexSearcher = memoryIndex.createSearcher();
+
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer));
+ writer.addDocument(doc);
+ writer.close();
+ IndexReader controlIndexReader = DirectoryReader.open(dir);
+ IndexSearcher controlIndexSearcher = new IndexSearcher(controlIndexReader);
+
+ Supplier<Integer> valueSupplier = () -> randomValues.get(random().nextInt(randomValues.size()));
+ Query[] queries = new Query[] {
+ IntPoint.newExactQuery("int", valueSupplier.get()),
+ LongPoint.newExactQuery("long", valueSupplier.get()),
+ FloatPoint.newExactQuery("float", valueSupplier.get()),
+ DoublePoint.newExactQuery("double", valueSupplier.get()),
+ IntPoint.newSetQuery("int", valueSupplier.get(), valueSupplier.get()),
+ LongPoint.newSetQuery("long", valueSupplier.get(), valueSupplier.get()),
+ FloatPoint.newSetQuery("float", valueSupplier.get(), valueSupplier.get()),
+ DoublePoint.newSetQuery("double", valueSupplier.get(), valueSupplier.get()),
+ IntPoint.newRangeQuery("int", valueSupplier.get(), valueSupplier.get()),
+ LongPoint.newRangeQuery("long", valueSupplier.get(), valueSupplier.get()),
+ FloatPoint.newRangeQuery("float", valueSupplier.get(), valueSupplier.get()),
+ DoublePoint.newRangeQuery("double", valueSupplier.get(), valueSupplier.get())
+ };
+ for (Query query : queries) {
+ assertEquals(controlIndexSearcher.count(query), controlIndexSearcher.count(query));
+ }
+
+ memoryIndexSearcher.getIndexReader().close();
+ controlIndexReader.close();
+ dir.close();
+ }
+
public void testDuellMemIndex() throws IOException {
LineFileDocs lineFileDocs = new LineFileDocs(random());
int numDocs = atLeast(10);
[24/50] lucene-solr:jira/SOLR-445: re-enable accidentally turned off
test evilness
Posted by ho...@apache.org.
re-enable accidentally turned off test evilness
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/56ca641b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/56ca641b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/56ca641b
Branch: refs/heads/jira/SOLR-445
Commit: 56ca641b5b50dd9133753410d40dda0632e873f5
Parents: cf3eea2
Author: Mike McCandless <mi...@apache.org>
Authored: Tue Mar 15 05:18:53 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Tue Mar 15 05:18:53 2016 -0400
----------------------------------------------------------------------
lucene/core/src/test/org/apache/lucene/index/TestPointValues.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/56ca641b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
index 0946234..49cbc2a 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
@@ -501,7 +501,7 @@ public class TestPointValues extends LuceneTestCase {
doc.add(new IntPoint("int", 17));
for(int i=0;i<300000;i++) {
w.addDocument(doc);
- if (false && random().nextInt(1000) == 17) {
+ if (random().nextInt(1000) == 17) {
w.commit();
}
}
[08/50] lucene-solr:jira/SOLR-445: fix int overflow
Posted by ho...@apache.org.
fix int overflow
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fcd90b9b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fcd90b9b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fcd90b9b
Branch: refs/heads/jira/SOLR-445
Commit: fcd90b9ba649c88d4dc74b51b11335bb4fc9af88
Parents: fa97007
Author: Mike McCandless <mi...@apache.org>
Authored: Sun Mar 13 06:28:18 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Sun Mar 13 06:28:18 2016 -0400
----------------------------------------------------------------------
lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fcd90b9b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 765b01c..33d7bc4 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -1177,7 +1177,7 @@ public class BKDWriter implements Closeable {
PointReader reader = slices[dim].writer.getReader(slices[dim].start);) {
// Partition this source according to how the splitDim split the values:
- int nextRightCount = 0;
+ long nextRightCount = 0;
for (long i=0;i<source.count;i++) {
boolean result = reader.next();
assert result;
[43/50] lucene-solr:jira/SOLR-445: SOLR-8745: Move CHANGES.txt entry
to 6.1
Posted by ho...@apache.org.
SOLR-8745: Move CHANGES.txt entry to 6.1
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4fbfeb01
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4fbfeb01
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4fbfeb01
Branch: refs/heads/jira/SOLR-445
Commit: 4fbfeb01230429b073039b4d16b8871c1854f413
Parents: 85945ef
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Thu Mar 17 20:18:04 2016 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Thu Mar 17 20:18:04 2016 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fbfeb01/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ef5d422..1be92c9 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -56,6 +56,9 @@ Optimizations
* SOLR-8722: Don't force a full ZkStateReader refresh on every Overseer operation.
(Scott Blum via shalin)
+* SOLR-8745: Deprecate costly ZkStateReader.updateClusterState(), replace with a narrow
+ forceUpdateCollection(collection) (Scott Blum via shalin)
+
Other Changes
----------------------
* SOLR-7516: Improve javadocs for JavaBinCodec, ObjectResolver and enforce the single-usage policy.
@@ -346,9 +349,6 @@ Optimizations
* SOLR-8720: ZkController#publishAndWaitForDownStates should use #publishNodeAsDown. (Mark Miller)
-* SOLR-8745: Deprecate costly ZkStateReader.updateClusterState(), replace with a narrow
- forceUpdateCollection(collection) (Scott Blum via shalin)
-
Other Changes
----------------------
[14/50] lucene-solr:jira/SOLR-445: LUCENE-7097: let IntroSorter go 2X
deeper in quicksort before switching to heapsort
Posted by ho...@apache.org.
LUCENE-7097: let IntroSorter go 2X deeper in quicksort before switching to heapsort
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8cbe4713
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8cbe4713
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8cbe4713
Branch: refs/heads/jira/SOLR-445
Commit: 8cbe4713775565a3194e29b90747f59fe2ffe3f1
Parents: 3c7e55d
Author: Mike McCandless <mi...@apache.org>
Authored: Mon Mar 14 06:03:17 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Mon Mar 14 06:03:17 2016 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
lucene/core/src/java/org/apache/lucene/util/IntroSorter.java | 6 +-----
2 files changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8cbe4713/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index cd9fac2..4998eb0 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -21,6 +21,9 @@ Optimizations
* LUCENE-7099: LatLonPoint's newDistanceQuery supports two-phase
iteration. (Robert Muir)
+* LUCENE-7097: IntroSorter now recurses to 2 * log_2(count) quicksort
+ stack depth before switching to heapsort (Adrien Grand, Mike McCandless)
+
Other
* LUCENE-7087: Let MemoryIndex#fromDocument(...) accept 'Iterable<? extends IndexableField>'
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8cbe4713/lucene/core/src/java/org/apache/lucene/util/IntroSorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/IntroSorter.java b/lucene/core/src/java/org/apache/lucene/util/IntroSorter.java
index d9cdd62..498c06a 100644
--- a/lucene/core/src/java/org/apache/lucene/util/IntroSorter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/IntroSorter.java
@@ -28,17 +28,13 @@ package org.apache.lucene.util;
*/
public abstract class IntroSorter extends Sorter {
- static int ceilLog2(int n) {
- return Integer.SIZE - Integer.numberOfLeadingZeros(n - 1);
- }
-
/** Create a new {@link IntroSorter}. */
public IntroSorter() {}
@Override
public final void sort(int from, int to) {
checkRange(from, to);
- quicksort(from, to, ceilLog2(to - from));
+ quicksort(from, to, 2 * MathUtil.log(to - from, 2));
}
void quicksort(int from, int to, int maxDepth) {
[15/50] lucene-solr:jira/SOLR-445: fix test bug: these tests expect
only one segment
Posted by ho...@apache.org.
fix test bug: these tests expect only one segment
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f706c9de
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f706c9de
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f706c9de
Branch: refs/heads/jira/SOLR-445
Commit: f706c9de82c7fe92940965512654e871e0b6e645
Parents: 8cbe471
Author: Mike McCandless <mi...@apache.org>
Authored: Mon Mar 14 10:21:57 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Mon Mar 14 10:21:57 2016 -0400
----------------------------------------------------------------------
.../test/org/apache/lucene/search/spans/TestSpanCollection.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f706c9de/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanCollection.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanCollection.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanCollection.java
index dfc0439..7b90b2b 100644
--- a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanCollection.java
+++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanCollection.java
@@ -61,7 +61,7 @@ public class TestSpanCollection extends LuceneTestCase {
super.setUp();
directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
- newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE));
+ newIndexWriterConfig(new MockAnalyzer(random())));
for (int i = 0; i < docFields.length; i++) {
Document doc = new Document();
doc.add(newField(FIELD, docFields[i], OFFSETS));
[26/50] lucene-solr:jira/SOLR-445: LUCENE-7101: OfflineSorter had
O(N^2) merge cost, and used too many temporary file descriptors,
for large sorts
Posted by ho...@apache.org.
LUCENE-7101: OfflineSorter had O(N^2) merge cost, and used too many temporary file descriptors, for large sorts
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/82c06190
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/82c06190
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/82c06190
Branch: refs/heads/jira/SOLR-445
Commit: 82c06190a35a8159288c2fb48d8d38d6d81dbbf2
Parents: 5801caa
Author: Mike McCandless <mi...@apache.org>
Authored: Tue Mar 15 07:36:02 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Tue Mar 15 07:36:02 2016 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 ++
.../org/apache/lucene/util/OfflineSorter.java | 44 ++++++++++++++------
.../org/apache/lucene/util/bkd/BKDWriter.java | 14 +++----
.../org/apache/lucene/index/Test2BPoints.java | 21 +---------
.../apache/lucene/util/TestOfflineSorter.java | 2 +-
.../apache/lucene/util/bkd/Test2BBKDPoints.java | 4 +-
6 files changed, 44 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/82c06190/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index db08eb3..61737d9 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -196,6 +196,9 @@ Bug Fixes
On top of that with score mode average, the explain would fail with a NPE.
(Martijn van Groningen)
+* LUCENE-7101: OfflineSorter had O(N^2) merge cost, and used too many
+ temporary file descriptors, for large sorts (Mike McCandless)
+
Other
* LUCENE-7035: Upgrade icu4j to 56.1/unicode 8. (Robert Muir)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/82c06190/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java b/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java
index 18e421b..3a22e33 100644
--- a/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java
@@ -64,7 +64,7 @@ public class OfflineSorter {
/**
* Maximum number of temporary files before doing an intermediate merge.
*/
- public final static int MAX_TEMPFILES = 128;
+ public final static int MAX_TEMPFILES = 10;
private final Directory dir;
@@ -232,6 +232,7 @@ public class OfflineSorter {
sortInfo.totalTime = System.currentTimeMillis();
List<String> segments = new ArrayList<>();
+ int[] levelCounts = new int[1];
// So we can remove any partially written temp files on exception:
TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(dir);
@@ -244,15 +245,26 @@ public class OfflineSorter {
segments.add(sortPartition(trackingDir));
sortInfo.tempMergeFiles++;
sortInfo.lineCount += lineCount;
+ levelCounts[0]++;
- // Handle intermediate merges.
- if (segments.size() == maxTempFiles) {
+ // Handle intermediate merges; we need a while loop to "cascade" the merge when necessary:
+ int mergeLevel = 0;
+ while (levelCounts[mergeLevel] == maxTempFiles) {
mergePartitions(trackingDir, segments);
+ if (mergeLevel+2 > levelCounts.length) {
+ levelCounts = ArrayUtil.grow(levelCounts, mergeLevel+2);
+ }
+ levelCounts[mergeLevel+1]++;
+ levelCounts[mergeLevel] = 0;
+ mergeLevel++;
}
}
+
+ // TODO: we shouldn't have to do this? Can't we return a merged reader to
+ // the caller, who often consumes the result just once, instead?
- // Merge the partitions to the output file with a priority queue.
- if (segments.size() > 1) {
+ // Merge all partitions down to 1 (basically a forceMerge(1)):
+ while (segments.size() > 1) {
mergePartitions(trackingDir, segments);
}
@@ -304,19 +316,25 @@ public class OfflineSorter {
}
}
- /** Merge a list of sorted temporary files (partitions) into an output file. Note that this closes the
- * incoming {@link IndexOutput}. */
+ /** Merge the most recent {@code maxTempFile} partitions into a new partition. */
void mergePartitions(Directory trackingDir, List<String> segments) throws IOException {
long start = System.currentTimeMillis();
- PriorityQueue<FileAndTop> queue = new PriorityQueue<FileAndTop>(segments.size()) {
+ List<String> segmentsToMerge;
+ if (segments.size() > maxTempFiles) {
+ segmentsToMerge = segments.subList(segments.size() - maxTempFiles, segments.size());
+ } else {
+ segmentsToMerge = segments;
+ }
+
+ PriorityQueue<FileAndTop> queue = new PriorityQueue<FileAndTop>(segmentsToMerge.size()) {
@Override
protected boolean lessThan(FileAndTop a, FileAndTop b) {
return comparator.compare(a.current.get(), b.current.get()) < 0;
}
};
- ByteSequencesReader[] streams = new ByteSequencesReader[segments.size()];
+ ByteSequencesReader[] streams = new ByteSequencesReader[segmentsToMerge.size()];
String newSegmentName = null;
@@ -326,8 +344,8 @@ public class OfflineSorter {
newSegmentName = out.getName();
// Open streams and read the top for each file
- for (int i = 0; i < segments.size(); i++) {
- streams[i] = getReader(dir.openInput(segments.get(i), IOContext.READONCE));
+ for (int i = 0; i < segmentsToMerge.size(); i++) {
+ streams[i] = getReader(dir.openInput(segmentsToMerge.get(i), IOContext.READONCE));
BytesRefBuilder bytes = new BytesRefBuilder();
boolean result = streams[i].read(bytes);
assert result;
@@ -354,9 +372,9 @@ public class OfflineSorter {
IOUtils.close(streams);
}
- IOUtils.deleteFiles(trackingDir, segments);
+ IOUtils.deleteFiles(trackingDir, segmentsToMerge);
- segments.clear();
+ segmentsToMerge.clear();
segments.add(newSegmentName);
sortInfo.tempMergeFiles++;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/82c06190/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index c5cdc30..10d97e3 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -212,11 +212,11 @@ public class BKDWriter implements Closeable {
}
}
- /** If the current segment has too many points then we switchover to temp files / offline sort. */
- private void switchToOffline() throws IOException {
+ /** If the current segment has too many points then we spill over to temp files / offline sort. */
+ private void spillToOffline() throws IOException {
// For each .add we just append to this input file, then in .finish we sort this input and resursively build the tree:
- offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds, "switch");
+ offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds, "spill");
tempInput = offlinePointWriter.out;
PointReader reader = heapPointWriter.getReader(0);
for(int i=0;i<pointCount;i++) {
@@ -235,7 +235,7 @@ public class BKDWriter implements Closeable {
if (pointCount >= maxPointsSortInHeap) {
if (offlinePointWriter == null) {
- switchToOffline();
+ spillToOffline();
}
offlinePointWriter.append(packedValue, pointCount, docID);
} else {
@@ -733,7 +733,7 @@ public class BKDWriter implements Closeable {
// TODO: this is sort of sneaky way to get the final OfflinePointWriter from OfflineSorter:
IndexOutput[] lastWriter = new IndexOutput[1];
- OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix, cmp, OfflineSorter.BufferSize.megabytes(Math.max(1, (long) maxMBSortInHeap)), OfflineSorter.MAX_TEMPFILES) {
+ OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix + "_bkd" + dim, cmp, OfflineSorter.BufferSize.megabytes(Math.max(1, (long) maxMBSortInHeap)), OfflineSorter.MAX_TEMPFILES) {
/** We write/read fixed-byte-width file that {@link OfflinePointReader} can read. */
@Override
@@ -742,9 +742,7 @@ public class BKDWriter implements Closeable {
return new ByteSequencesWriter(out) {
@Override
public void write(byte[] bytes, int off, int len) throws IOException {
- if (len != bytesPerDoc) {
- throw new IllegalArgumentException("len=" + len + " bytesPerDoc=" + bytesPerDoc);
- }
+ assert len == bytesPerDoc: "len=" + len + " bytesPerDoc=" + bytesPerDoc;
out.writeBytes(bytes, off, len);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/82c06190/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
index 5766b91..f15ba19 100644
--- a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
+++ b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
@@ -49,7 +49,6 @@ import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
public class Test2BPoints extends LuceneTestCase {
public void test1D() throws Exception {
Directory dir = FSDirectory.open(createTempDir("2BPoints1D"));
- System.out.println("DIR: " + ((FSDirectory) dir).getDirectory());
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()))
.setCodec(getCodec())
@@ -144,24 +143,6 @@ public class Test2BPoints extends LuceneTestCase {
}
private static Codec getCodec() {
-
- return new FilterCodec("Lucene60", Codec.forName("Lucene60")) {
- @Override
- public PointsFormat pointsFormat() {
- return new PointsFormat() {
- @Override
- public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException {
- int maxPointsInLeafNode = 1024;
- double maxMBSortInHeap = 256.0;
- return new Lucene60PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap);
- }
-
- @Override
- public PointsReader fieldsReader(SegmentReadState readState) throws IOException {
- return new Lucene60PointsReader(readState);
- }
- };
- }
- };
+ return Codec.forName("Lucene60");
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/82c06190/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java b/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
index 5322531..b45a3bb 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
@@ -82,7 +82,7 @@ public class TestOfflineSorter extends LuceneTestCase {
try (Directory dir = newDirectory()) {
SortInfo sortInfo = checkSort(dir, new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), OfflineSorter.MAX_TEMPFILES),
generateRandom((int)OfflineSorter.MB * 20));
- assertEquals(1, sortInfo.mergeRounds);
+ assertEquals(3, sortInfo.mergeRounds);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/82c06190/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
index 4df2ebe..cfb98b0 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
@@ -55,7 +55,7 @@ public class Test2BBKDPoints extends LuceneTestCase {
final int numDocs = (Integer.MAX_VALUE / 26) + 100;
- BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, Long.BYTES, 1024, 256, 26L * numDocs);
+ BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, Long.BYTES, 26L * numDocs);
int counter = 0;
byte[] packedBytes = new byte[Long.BYTES];
for (int docID = 0; docID < numDocs; docID++) {
@@ -88,7 +88,7 @@ public class Test2BBKDPoints extends LuceneTestCase {
final int numDocs = (Integer.MAX_VALUE / 26) + 100;
- BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES, 1024, 256, 26L * numDocs);
+ BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES, 26L * numDocs);
int counter = 0;
byte[] packedBytes = new byte[2*Long.BYTES];
for (int docID = 0; docID < numDocs; docID++) {
[44/50] lucene-solr:jira/SOLR-445: More javadocs about exclusive
bounds.
Posted by ho...@apache.org.
More javadocs about exclusive bounds.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c1e95d7b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c1e95d7b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c1e95d7b
Branch: refs/heads/jira/SOLR-445
Commit: c1e95d7b4daf60a558c2288b36adbd5632fe7b40
Parents: 4fbfeb0
Author: Adrien Grand <jp...@gmail.com>
Authored: Thu Mar 17 16:25:05 2016 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Thu Mar 17 16:25:27 2016 +0100
----------------------------------------------------------------------
.../src/java/org/apache/lucene/document/DoublePoint.java | 8 ++++++--
.../core/src/java/org/apache/lucene/document/FloatPoint.java | 8 ++++++--
.../core/src/java/org/apache/lucene/document/IntPoint.java | 8 +++++---
.../core/src/java/org/apache/lucene/document/LongPoint.java | 6 ++++--
4 files changed, 21 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1e95d7b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java
index 1133b22..1eb1d7e 100644
--- a/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java
+++ b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java
@@ -172,7 +172,9 @@ public final class DoublePoint extends Field {
* {@link #newRangeQuery(String, double[], double[])} instead.
* <p>
* You can have half-open ranges (which are in fact </≤ or >/≥ queries)
- * by setting {@code lowerValue = Double.NEGATIVE_INFINITY} or {@code upperValue = Double.POSITIVE_INFINITY}.
+ * by setting {@code lowerValue = Double.NEGATIVE_INFINITY} or {@code upperValue = Double.POSITIVE_INFINITY}.
+ * <p> Ranges are inclusive. For exclusive ranges, pass {@code Math#nextUp(lowerValue)}
+ * or {@code Math.nextDown(upperValue)}.
* <p>
* Range comparisons are consistent with {@link Double#compareTo(Double)}.
*
@@ -190,7 +192,9 @@ public final class DoublePoint extends Field {
* Create a range query for n-dimensional double values.
* <p>
* You can have half-open ranges (which are in fact </≤ or >/≥ queries)
- * by setting {@code lowerValue[i] = Double.NEGATIVE_INFINITY} or {@code upperValue[i] = Double.POSITIVE_INFINITY}.
+ * by setting {@code lowerValue[i] = Double.NEGATIVE_INFINITY} or {@code upperValue[i] = Double.POSITIVE_INFINITY}.
+ * <p> Ranges are inclusive. For exclusive ranges, pass {@code Math#nextUp(lowerValue[i])}
+ * or {@code Math.nextDown(upperValue[i])}.
* <p>
* Range comparisons are consistent with {@link Double#compareTo(Double)}.
*
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1e95d7b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java
index 3d110db..08192e5 100644
--- a/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java
+++ b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java
@@ -172,7 +172,9 @@ public final class FloatPoint extends Field {
* {@link #newRangeQuery(String, float[], float[])} instead.
* <p>
* You can have half-open ranges (which are in fact </≤ or >/≥ queries)
- * by setting {@code lowerValue = Float.NEGATIVE_INFINITY} or {@code upperValue = Float.POSITIVE_INFINITY}.
+ * by setting {@code lowerValue = Float.NEGATIVE_INFINITY} or {@code upperValue = Float.POSITIVE_INFINITY}.
+ * <p> Ranges are inclusive. For exclusive ranges, pass {@code Math#nextUp(lowerValue)}
+ * or {@code Math.nextDown(upperValue)}.
* <p>
* Range comparisons are consistent with {@link Float#compareTo(Float)}.
*
@@ -190,7 +192,9 @@ public final class FloatPoint extends Field {
* Create a range query for n-dimensional float values.
* <p>
* You can have half-open ranges (which are in fact </≤ or >/≥ queries)
- * by setting {@code lowerValue[i] = Float.NEGATIVE_INFINITY} or {@code upperValue[i] = Float.POSITIVE_INFINITY}.
+ * by setting {@code lowerValue[i] = Float.NEGATIVE_INFINITY} or {@code upperValue[i] = Float.POSITIVE_INFINITY}.
+ * <p> Ranges are inclusive. For exclusive ranges, pass {@code Math#nextUp(lowerValue[i])}
+ * or {@code Math.nextDown(upperValue[i])}.
* <p>
* Range comparisons are consistent with {@link Float#compareTo(Float)}.
*
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1e95d7b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/IntPoint.java b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java
index 53ae3d3..c40a4b7 100644
--- a/lucene/core/src/java/org/apache/lucene/document/IntPoint.java
+++ b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java
@@ -172,9 +172,10 @@ public final class IntPoint extends Field {
* {@link #newRangeQuery(String, int[], int[])} instead.
* <p>
* You can have half-open ranges (which are in fact </≤ or >/≥ queries)
- * by setting {@code lowerValue = Integer.MIN_VALUE} or {@code upperValue = Integer.MAX_VALUE}.
+ * by setting {@code lowerValue = Integer.MIN_VALUE} or {@code upperValue = Integer.MAX_VALUE}.
* <p>
- * Ranges are inclusive. For exclusive ranges, pass {@code lowerValue + 1} or {@code upperValue - 1}
+ * Ranges are inclusive. For exclusive ranges, pass {@code Math.addExact(lowerValue, 1)}
+ * or {@code Math.addExact(upperValue, -1)}.
*
* @param field field name. must not be {@code null}.
* @param lowerValue lower portion of the range (inclusive).
@@ -192,7 +193,8 @@ public final class IntPoint extends Field {
* You can have half-open ranges (which are in fact </≤ or >/≥ queries)
* by setting {@code lowerValue[i] = Integer.MIN_VALUE} or {@code upperValue[i] = Integer.MAX_VALUE}.
* <p>
- * Ranges are inclusive. For exclusive ranges, pass {@code lowerValue[i] + 1} or {@code upperValue[i] - 1}
+ * Ranges are inclusive. For exclusive ranges, pass {@code Math.addExact(lowerValue[i], 1)}
+ * or {@code Math.addExact(upperValue[i], -1)}.
*
* @param field field name. must not be {@code null}.
* @param lowerValue lower portion of the range (inclusive). must not be {@code null}.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1e95d7b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/LongPoint.java b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java
index c4fd887..46bf93c 100644
--- a/lucene/core/src/java/org/apache/lucene/document/LongPoint.java
+++ b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java
@@ -174,7 +174,8 @@ public final class LongPoint extends Field {
* You can have half-open ranges (which are in fact </≤ or >/≥ queries)
* by setting {@code lowerValue = Long.MIN_VALUE} or {@code upperValue = Long.MAX_VALUE}.
* <p>
- * Ranges are inclusive. For exclusive ranges, pass {@code lowerValue + 1} or {@code upperValue - 1}
+ * Ranges are inclusive. For exclusive ranges, pass {@code Math.addExact(lowerValue, 1)}
+ * or {@code Math.addExact(upperValue, -1)}.
*
* @param field field name. must not be {@code null}.
* @param lowerValue lower portion of the range (inclusive).
@@ -192,7 +193,8 @@ public final class LongPoint extends Field {
* You can have half-open ranges (which are in fact </≤ or >/≥ queries)
* by setting {@code lowerValue[i] = Long.MIN_VALUE} or {@code upperValue[i] = Long.MAX_VALUE}.
* <p>
- * Ranges are inclusive. For exclusive ranges, pass {@code lowerValue[i] + 1} or {@code upperValue[i] - 1}
+ * Ranges are inclusive. For exclusive ranges, pass {@code Math.addExact(lowerValue[i], 1)}
+ * or {@code Math.addExact(upperValue[i], -1)}.
*
* @param field field name. must not be {@code null}.
* @param lowerValue lower portion of the range (inclusive). must not be {@code null}.
[31/50] lucene-solr:jira/SOLR-445: SOLR-8849: improve reproducibility
in random order of chaosmonkey actions
Posted by ho...@apache.org.
SOLR-8849: improve reproducibility in random order of chaosmonkey actions
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0f78235b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0f78235b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0f78235b
Branch: refs/heads/jira/SOLR-445
Commit: 0f78235b9450ed7a81313dd9c7b9d7dfa4b57ee3
Parents: 30a77b7
Author: Chris Hostetter <ho...@apache.org>
Authored: Tue Mar 15 11:04:33 2016 -0700
Committer: Chris Hostetter <ho...@apache.org>
Committed: Tue Mar 15 11:17:45 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/solr/cloud/ChaosMonkey.java | 100 +++++++++++--------
1 file changed, 56 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0f78235b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
index 511fdf3..93a670e 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
@@ -85,6 +85,13 @@ public class ChaosMonkey {
private List<CloudJettyRunner> deadPool = new ArrayList<>();
private Thread monkeyThread;
+
+ /**
+ * Our own Random, seeded from LuceneTestCase on init, so that we can produce a consistent sequence
+ * of random chaos regardless of if/how othe threads access the test randomness in other threads
+ * @see LuceneTestCase#random()
+ */
+ private final Random chaosRandom;
public ChaosMonkey(ZkTestServer zkServer, ZkStateReader zkStateReader,
String collection, Map<String,List<CloudJettyRunner>> shardToJetty,
@@ -94,22 +101,22 @@ public class ChaosMonkey {
this.zkServer = zkServer;
this.zkStateReader = zkStateReader;
this.collection = collection;
+ this.chaosRandom = new Random(LuceneTestCase.random().nextLong());
if (!MONKEY_ENABLED) {
monkeyLog("The Monkey is Disabled and will not run");
return;
}
- Random random = LuceneTestCase.random();
if (EXP != null) {
expireSessions = EXP;
} else {
- expireSessions = random.nextBoolean();
+ expireSessions = chaosRandom.nextBoolean();
}
if (CONN_LOSS != null) {
causeConnectionLoss = CONN_LOSS;
} else {
- causeConnectionLoss = random.nextBoolean();
+ causeConnectionLoss = chaosRandom.nextBoolean();
}
@@ -326,7 +333,7 @@ public class ChaosMonkey {
List<String> sliceKeyList = new ArrayList<>(slices.size());
sliceKeyList.addAll(slices.keySet());
- String sliceName = sliceKeyList.get(LuceneTestCase.random().nextInt(sliceKeyList.size()));
+ String sliceName = sliceKeyList.get(chaosRandom.nextInt(sliceKeyList.size()));
return sliceName;
}
@@ -365,8 +372,7 @@ public class ChaosMonkey {
return null;
}
- Random random = LuceneTestCase.random();
- int chance = random.nextInt(10);
+ int chance = chaosRandom.nextInt(10);
CloudJettyRunner cjetty;
if (chance <= 5 && aggressivelyKillLeaders) {
// if killLeader, really aggressively go after leaders
@@ -374,7 +380,7 @@ public class ChaosMonkey {
} else {
// get random shard
List<CloudJettyRunner> jetties = shardToJetty.get(slice);
- int index = random.nextInt(jetties.size());
+ int index = chaosRandom.nextInt(jetties.size());
cjetty = jetties.get(index);
ZkNodeProps leader = null;
@@ -457,7 +463,7 @@ public class ChaosMonkey {
monkeyLog("starting");
- if (LuceneTestCase.random().nextBoolean()) {
+ if (chaosRandom.nextBoolean()) {
monkeyLog("Jetty will not commit on close");
DirectUpdateHandler2.commitOnClose = false;
}
@@ -474,43 +480,9 @@ public class ChaosMonkey {
while (!stop) {
try {
- Random random = LuceneTestCase.random();
- Thread.sleep(random.nextInt(roundPauseUpperLimit));
- if (random.nextBoolean()) {
- if (!deadPool.isEmpty()) {
- int index = random.nextInt(deadPool.size());
- JettySolrRunner jetty = deadPool.get(index).jetty;
- if (jetty.isStopped() && !ChaosMonkey.start(jetty)) {
- continue;
- }
- //System.out.println("started on port:" + jetty.getLocalPort());
- deadPool.remove(index);
- starts.incrementAndGet();
- continue;
- }
- }
-
- int rnd = random.nextInt(10);
+ Thread.sleep(chaosRandom.nextInt(roundPauseUpperLimit));
- if (expireSessions && rnd < EXPIRE_PERCENT) {
- expireRandomSession();
- }
-
- if (causeConnectionLoss && rnd < CONLOSS_PERCENT) {
- randomConnectionLoss();
- }
-
- CloudJettyRunner cjetty;
- if (random.nextBoolean()) {
- cjetty = stopRandomShard();
- } else {
- cjetty = killRandomShard();
- }
- if (cjetty == null) {
- // we cannot kill
- } else {
- deadPool.add(cjetty);
- }
+ causeSomeChaos();
} catch (InterruptedException e) {
//
@@ -549,6 +521,46 @@ public class ChaosMonkey {
}
}
+ /**
+ * causes some randomly selected chaos
+ */
+ public void causeSomeChaos() throws Exception {
+ if (chaosRandom.nextBoolean()) {
+ if (!deadPool.isEmpty()) {
+ int index = chaosRandom.nextInt(deadPool.size());
+ JettySolrRunner jetty = deadPool.get(index).jetty;
+ if (jetty.isStopped() && !ChaosMonkey.start(jetty)) {
+ return;
+ }
+ deadPool.remove(index);
+ starts.incrementAndGet();
+ return;
+ }
+ }
+
+ int rnd = chaosRandom.nextInt(10);
+
+ if (expireSessions && rnd < EXPIRE_PERCENT) {
+ expireRandomSession();
+ }
+
+ if (causeConnectionLoss && rnd < CONLOSS_PERCENT) {
+ randomConnectionLoss();
+ }
+
+ CloudJettyRunner cjetty;
+ if (chaosRandom.nextBoolean()) {
+ cjetty = stopRandomShard();
+ } else {
+ cjetty = killRandomShard();
+ }
+ if (cjetty == null) {
+ // we cannot kill
+ } else {
+ deadPool.add(cjetty);
+ }
+ }
+
public int getStarts() {
return starts.get();
}
[19/50] lucene-solr:jira/SOLR-445: LUCENE-7102:
LatLonPoint.newDistanceSort fails with "sort missing first"
Posted by ho...@apache.org.
LUCENE-7102: LatLonPoint.newDistanceSort fails with "sort missing first"
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0f949c81
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0f949c81
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0f949c81
Branch: refs/heads/jira/SOLR-445
Commit: 0f949c815343c853499d518e7d565d642d93ce63
Parents: 80fe00b
Author: Robert Muir <rm...@apache.org>
Authored: Mon Mar 14 14:08:25 2016 -0400
Committer: Robert Muir <rm...@apache.org>
Committed: Mon Mar 14 14:08:25 2016 -0400
----------------------------------------------------------------------
.../org/apache/lucene/document/LatLonPoint.java | 4 +-
.../document/LatLonPointDistanceComparator.java | 84 +++++++-------
.../lucene/document/LatLonPointSortField.java | 6 +-
.../document/TestLatLonPointDistanceSort.java | 111 +++++++++++++++++--
4 files changed, 152 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0f949c81/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
index ebf850c..9677baa 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
@@ -333,8 +333,8 @@ public class LatLonPoint extends Field {
* the hits contains a Double instance with the distance in meters.
* <p>
* If a document is missing the field, then by default it is treated as having {@link Double#POSITIVE_INFINITY} distance
- * (missing last). You can change this by calling {@link SortField#setMissingValue(Object)} on the returned SortField
- * to a different Double value.
+ * (missing values sort last). You can change this to sort missing values first by calling
+ * {@link SortField#setMissingValue(Object) setMissingValue(Double.NEGATIVE_INFINITY)} on the returned SortField.
* <p>
* If a document contains multiple values for the field, the <i>closest</i> distance to the location is used.
* <p>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0f949c81/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
index e64f4b0..86c9134 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java
@@ -89,45 +89,53 @@ class LatLonPointDistanceComparator extends FieldComparator<Double> implements L
// sampling if we get called way too much: don't make gobs of bounding
// boxes if comparator hits a worst case order (e.g. backwards distance order)
if (setBottomCounter < 1024 || (setBottomCounter & 0x3F) == 0x3F) {
- GeoRect box = GeoUtils.circleToBBox(longitude, latitude, bottom);
- // pre-encode our box to our integer encoding, so we don't have to decode
- // to double values for uncompetitive hits. This has some cost!
- int minLatEncoded = LatLonPoint.encodeLatitude(box.minLat);
- int maxLatEncoded = LatLonPoint.encodeLatitude(box.maxLat);
- int minLonEncoded = LatLonPoint.encodeLongitude(box.minLon);
- int maxLonEncoded = LatLonPoint.encodeLongitude(box.maxLon);
- // be sure to not introduce quantization error in our optimization, just
- // round up our encoded box safely in all directions.
- if (minLatEncoded != Integer.MIN_VALUE) {
- minLatEncoded--;
- }
- if (minLonEncoded != Integer.MIN_VALUE) {
- minLonEncoded--;
- }
- if (maxLatEncoded != Integer.MAX_VALUE) {
- maxLatEncoded++;
- }
- if (maxLonEncoded != Integer.MAX_VALUE) {
- maxLonEncoded++;
- }
- crossesDateLine = box.crossesDateline();
- // crosses dateline: split
- if (crossesDateLine) {
- // box1
- minLon = Integer.MIN_VALUE;
- maxLon = maxLonEncoded;
- minLat = minLatEncoded;
- maxLat = maxLatEncoded;
- // box2
- minLon2 = minLonEncoded;
- maxLon2 = Integer.MAX_VALUE;
- minLat2 = minLatEncoded;
- maxLat2 = maxLatEncoded;
+ // don't pass infinite values to circleToBBox: just make a complete box.
+ if (bottom == missingValue) {
+ minLat = minLon = Integer.MIN_VALUE;
+ maxLat = maxLon = Integer.MAX_VALUE;
+ crossesDateLine = false;
} else {
- minLon = minLonEncoded;
- maxLon = maxLonEncoded;
- minLat = minLatEncoded;
- maxLat = maxLatEncoded;
+ assert Double.isFinite(bottom);
+ GeoRect box = GeoUtils.circleToBBox(longitude, latitude, bottom);
+ // pre-encode our box to our integer encoding, so we don't have to decode
+ // to double values for uncompetitive hits. This has some cost!
+ int minLatEncoded = LatLonPoint.encodeLatitude(box.minLat);
+ int maxLatEncoded = LatLonPoint.encodeLatitude(box.maxLat);
+ int minLonEncoded = LatLonPoint.encodeLongitude(box.minLon);
+ int maxLonEncoded = LatLonPoint.encodeLongitude(box.maxLon);
+ // be sure to not introduce quantization error in our optimization, just
+ // round up our encoded box safely in all directions.
+ if (minLatEncoded != Integer.MIN_VALUE) {
+ minLatEncoded--;
+ }
+ if (minLonEncoded != Integer.MIN_VALUE) {
+ minLonEncoded--;
+ }
+ if (maxLatEncoded != Integer.MAX_VALUE) {
+ maxLatEncoded++;
+ }
+ if (maxLonEncoded != Integer.MAX_VALUE) {
+ maxLonEncoded++;
+ }
+ crossesDateLine = box.crossesDateline();
+ // crosses dateline: split
+ if (crossesDateLine) {
+ // box1
+ minLon = Integer.MIN_VALUE;
+ maxLon = maxLonEncoded;
+ minLat = minLatEncoded;
+ maxLat = maxLatEncoded;
+ // box2
+ minLon2 = minLonEncoded;
+ maxLon2 = Integer.MAX_VALUE;
+ minLat2 = minLatEncoded;
+ maxLat2 = maxLatEncoded;
+ } else {
+ minLon = minLonEncoded;
+ maxLon = maxLonEncoded;
+ minLat = minLatEncoded;
+ maxLat = maxLatEncoded;
+ }
}
}
setBottomCounter++;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0f949c81/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java
index 9d3928e..da90b86 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java
@@ -62,7 +62,11 @@ final class LatLonPointSortField extends SortField {
}
if (missingValue.getClass() != Double.class)
throw new IllegalArgumentException("Missing value can only be of type java.lang.Double, but got " + missingValue.getClass());
- this.missingValue = missingValue;
+ Double value = (Double) missingValue;
+ if (!Double.isInfinite(value)) {
+ throw new IllegalArgumentException("Missing value can only be Double.NEGATIVE_INFINITY (missing values first) or Double.POSITIVE_INFINITY (missing values last), but got " + value);
+ }
+ this.missingValue = value;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0f949c81/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
index ea36ea6..a776b3f 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
@@ -73,6 +73,82 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
dir.close();
}
+ /** Add two points (one doc missing) and sort by distance */
+ public void testMissingLast() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+
+ // missing
+ Document doc = new Document();
+ iw.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new LatLonPoint("location", 40.718266, -74.007819));
+ iw.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new LatLonPoint("location", 40.7051157, -74.0088305));
+ iw.addDocument(doc);
+
+ IndexReader reader = iw.getReader();
+ IndexSearcher searcher = new IndexSearcher(reader);
+ iw.close();
+
+ Sort sort = new Sort(LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731));
+ TopDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort);
+
+ FieldDoc d = (FieldDoc) td.scoreDocs[0];
+ assertEquals(462.61748421408186D, (Double)d.fields[0], 0.0D);
+
+ d = (FieldDoc) td.scoreDocs[1];
+ assertEquals(1056.1630445911035D, (Double)d.fields[0], 0.0D);
+
+ d = (FieldDoc) td.scoreDocs[2];
+ assertEquals(Double.POSITIVE_INFINITY, (Double)d.fields[0], 0.0D);
+
+ reader.close();
+ dir.close();
+ }
+
+ /** Add two points (one doc missing) and sort by distance */
+ public void testMissingFirst() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+
+ // missing
+ Document doc = new Document();
+ iw.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new LatLonPoint("location", 40.718266, -74.007819));
+ iw.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new LatLonPoint("location", 40.7051157, -74.0088305));
+ iw.addDocument(doc);
+
+ IndexReader reader = iw.getReader();
+ IndexSearcher searcher = new IndexSearcher(reader);
+ iw.close();
+
+ SortField sortField = LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731);
+ sortField.setMissingValue(Double.NEGATIVE_INFINITY);
+ Sort sort = new Sort(sortField);
+ TopDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort);
+
+ FieldDoc d = (FieldDoc) td.scoreDocs[0];
+ assertEquals(Double.NEGATIVE_INFINITY, (Double)d.fields[0], 0.0D);
+
+ d = (FieldDoc) td.scoreDocs[1];
+ assertEquals(462.61748421408186D, (Double)d.fields[0], 0.0D);
+
+ d = (FieldDoc) td.scoreDocs[2];
+ assertEquals(1056.1630445911035D, (Double)d.fields[0], 0.0D);
+
+ reader.close();
+ dir.close();
+ }
+
/** Run a few iterations with just 10 docs, hopefully easy to debug */
public void testRandom() throws Exception {
for (int iters = 0; iters < 100; iters++) {
@@ -141,17 +217,20 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
for (int i = 0; i < numDocs; i++) {
- double latRaw = -90 + 180.0 * random().nextDouble();
- double lonRaw = -180 + 360.0 * random().nextDouble();
- // pre-normalize up front, so we can just use quantized value for testing and do simple exact comparisons
- double lat = LatLonPoint.decodeLatitude(LatLonPoint.encodeLatitude(latRaw));
- double lon = LatLonPoint.decodeLongitude(LatLonPoint.encodeLongitude(lonRaw));
Document doc = new Document();
doc.add(new StoredField("id", i));
doc.add(new NumericDocValuesField("id", i));
- doc.add(new LatLonPoint("field", lat, lon));
- doc.add(new StoredField("lat", lat));
- doc.add(new StoredField("lon", lon));
+ if (random().nextInt(10) > 7) {
+ double latRaw = -90 + 180.0 * random().nextDouble();
+ double lonRaw = -180 + 360.0 * random().nextDouble();
+ // pre-normalize up front, so we can just use quantized value for testing and do simple exact comparisons
+ double lat = LatLonPoint.decodeLatitude(LatLonPoint.encodeLatitude(latRaw));
+ double lon = LatLonPoint.decodeLongitude(LatLonPoint.encodeLongitude(lonRaw));
+
+ doc.add(new LatLonPoint("field", lat, lon));
+ doc.add(new StoredField("lat", lat));
+ doc.add(new StoredField("lon", lon));
+ } // otherwise "missing"
writer.addDocument(doc);
}
IndexReader reader = writer.getReader();
@@ -160,14 +239,20 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
for (int i = 0; i < numQueries; i++) {
double lat = -90 + 180.0 * random().nextDouble();
double lon = -180 + 360.0 * random().nextDouble();
+ double missingValue = random().nextBoolean() ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
Result expected[] = new Result[reader.maxDoc()];
for (int doc = 0; doc < reader.maxDoc(); doc++) {
Document targetDoc = reader.document(doc);
- double docLatitude = targetDoc.getField("lat").numericValue().doubleValue();
- double docLongitude = targetDoc.getField("lon").numericValue().doubleValue();
- double distance = GeoDistanceUtils.haversin(lat, lon, docLatitude, docLongitude);
+ final double distance;
+ if (targetDoc.getField("lat") == null) {
+ distance = missingValue; // missing
+ } else {
+ double docLatitude = targetDoc.getField("lat").numericValue().doubleValue();
+ double docLongitude = targetDoc.getField("lon").numericValue().doubleValue();
+ distance = GeoDistanceUtils.haversin(lat, lon, docLatitude, docLongitude);
+ }
int id = targetDoc.getField("id").numericValue().intValue();
expected[doc] = new Result(id, distance);
}
@@ -177,7 +262,9 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
// randomize the topN a bit
int topN = TestUtil.nextInt(random(), 1, reader.maxDoc());
// sort by distance, then ID
- Sort sort = new Sort(LatLonPoint.newDistanceSort("field", lat, lon),
+ SortField distanceSort = LatLonPoint.newDistanceSort("field", lat, lon);
+ distanceSort.setMissingValue(missingValue);
+ Sort sort = new Sort(distanceSort,
new SortField("id", SortField.Type.INT));
TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), topN, sort);
[38/50] lucene-solr:jira/SOLR-445: SOLR-8838: Returning non-stored
docValues is incorrect for negative floats and doubles.
Posted by ho...@apache.org.
SOLR-8838: Returning non-stored docValues is incorrect for negative floats and doubles.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6e55135b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6e55135b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6e55135b
Branch: refs/heads/jira/SOLR-445
Commit: 6e55135be3a8194db42a91de65d7746f0fc50332
Parents: e76fa56
Author: Steve Rowe <sa...@apache.org>
Authored: Wed Mar 16 18:56:23 2016 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Wed Mar 16 18:56:23 2016 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../apache/solr/search/SolrIndexSearcher.java | 4 +-
.../solr/schema/TestUseDocValuesAsStored.java | 158 +++++++++++++++----
3 files changed, 128 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6e55135b/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 945a123..ffcfabd 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -317,6 +317,9 @@ Bug Fixes
* SOLR-8835: JSON Facet API: fix faceting exception on multi-valued numeric fields that
have docValues. (yonik)
+* SOLR-8838: Returning non-stored docValues is incorrect for negative floats and doubles.
+ (Ishan Chattopadhyaya, Steve Rowe)
+
Optimizations
----------------------
* SOLR-7876: Speed up queries and operations that use many terms when timeAllowed has not been
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6e55135b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index 39fc8dd..4c9790a 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -816,9 +816,9 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
if (schemaField.getType() instanceof TrieIntField) {
newVal = val.intValue();
} else if (schemaField.getType() instanceof TrieFloatField) {
- newVal = NumericUtils.sortableIntToFloat(val.intValue());
+ newVal = Float.intBitsToFloat(val.intValue());
} else if (schemaField.getType() instanceof TrieDoubleField) {
- newVal = NumericUtils.sortableLongToDouble(val);
+ newVal = Double.longBitsToDouble(val);
} else if (schemaField.getType() instanceof TrieDateField) {
newVal = new Date(val);
} else if (schemaField.getType() instanceof EnumField) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6e55135b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java
index 839121a..ac245cd 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java
@@ -16,15 +16,36 @@
*/
package org.apache.solr.schema;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathFactory;
import java.io.File;
-import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.Month;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.TestUtil;
import org.apache.solr.core.AbstractBadConfigTestBase;
+import org.apache.solr.util.DOMUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
/**
* Tests the useDocValuesAsStored functionality.
@@ -38,30 +59,56 @@ public class TestUseDocValuesAsStored extends AbstractBadConfigTestBase {
private static final String collection = "collection1";
private static final String confDir = collection + "/conf";
-
+
+ private static final long START_RANDOM_EPOCH_MILLIS;
+ private static final long END_RANDOM_EPOCH_MILLIS;
+ private static final String[] SEVERITY;
+
+ // http://www.w3.org/TR/2006/REC-xml-20060816/#charsets
+ private static final String NON_XML_CHARS = "\u0000-\u0008\u000B-\u000C\u000E-\u001F\uFFFE\uFFFF";
+ // Avoid single quotes (problematic in XPath literals) and carriage returns (XML roundtripping fails)
+ private static final Pattern BAD_CHAR_PATTERN = Pattern.compile("[\'\r" + NON_XML_CHARS + "]");
+ private static final Pattern STORED_FIELD_NAME_PATTERN = Pattern.compile("_dv$");
+
+ static {
+ // Copy of DateTimeFormatter.ISO_INSTANT with fixed 3 digit milliseconds
+ START_RANDOM_EPOCH_MILLIS = LocalDateTime.of(1970, Month.JANUARY, 1, 0, 0)
+ .toInstant(ZoneOffset.UTC).toEpochMilli();
+ END_RANDOM_EPOCH_MILLIS = LocalDateTime.of(2030, Month.DECEMBER, 31, 23, 59, 59, 999_000_000)
+ .toInstant(ZoneOffset.UTC).toEpochMilli();
+ try {
+ DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
+ InputStream stream = TestUseDocValuesAsStored.class.getResourceAsStream("/solr/collection1/conf/enumsConfig.xml");
+ Document doc = builder.parse(new InputSource(IOUtils.getDecodingReader(stream, StandardCharsets.UTF_8)));
+ XPath xpath = XPathFactory.newInstance().newXPath();
+ NodeList nodes = (NodeList)xpath.evaluate
+ ("/enumsConfig/enum[@name='severity']/value", doc, XPathConstants.NODESET);
+ SEVERITY = new String[nodes.getLength()];
+ for (int i = 0 ; i < nodes.getLength() ; ++i) {
+ SEVERITY[i] = DOMUtil.getText(nodes.item(i));
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
@Before
private void initManagedSchemaCore() throws Exception {
tmpSolrHome = createTempDir().toFile();
tmpConfDir = new File(tmpSolrHome, confDir);
File testHomeConfDir = new File(TEST_HOME(), confDir);
FileUtils.copyFileToDirectory(new File(testHomeConfDir, "solrconfig-managed-schema.xml"), tmpConfDir);
- FileUtils.copyFileToDirectory(new File(testHomeConfDir, "solrconfig-basic.xml"), tmpConfDir);
FileUtils.copyFileToDirectory(new File(testHomeConfDir, "solrconfig.snippet.randomindexconfig.xml"), tmpConfDir);
- FileUtils.copyFileToDirectory(new File(testHomeConfDir, "schema-one-field-no-dynamic-field.xml"), tmpConfDir);
- FileUtils.copyFileToDirectory(new File(testHomeConfDir, "schema-one-field-no-dynamic-field-unique-key.xml"), tmpConfDir);
FileUtils.copyFileToDirectory(new File(testHomeConfDir, "enumsConfig.xml"), tmpConfDir);
FileUtils.copyFileToDirectory(new File(testHomeConfDir, "schema-non-stored-docvalues.xml"), tmpConfDir);
- FileUtils.copyFileToDirectory(new File(testHomeConfDir, "schema-minimal.xml"), tmpConfDir);
- FileUtils.copyFileToDirectory(new File(testHomeConfDir, "schema_codec.xml"), tmpConfDir);
- FileUtils.copyFileToDirectory(new File(testHomeConfDir, "schema-bm25.xml"), tmpConfDir);
-
+
// initCore will trigger an upgrade to managed schema, since the solrconfig has
// <schemaFactory class="ManagedIndexSchemaFactory" ... />
System.setProperty("enable.update.log", "false");
System.setProperty("managed.schema.mutable", "true");
initCore("solrconfig-managed-schema.xml", "schema-non-stored-docvalues.xml", tmpSolrHome.getPath());
}
-
+
@After
private void afterClass() throws Exception {
deleteCore();
@@ -116,31 +163,68 @@ public class TestUseDocValuesAsStored extends AbstractBadConfigTestBase {
}
@Test
- public void testSinglyValued() throws IOException {
- clearIndex();
- doTest("check string value is correct", "test_s_dvo", "str", "keyword");
- doTest("check int value is correct", "test_i_dvo", "int", "1234");
- doTest("check double value is correct", "test_d_dvo", "double", "1.234");
- doTest("check long value is correct", "test_l_dvo", "long", "12345");
- doTest("check float value is correct", "test_f_dvo", "float", "1.234");
- doTest("check dt value is correct", "test_dt_dvo", "date", "1976-07-04T12:08:56.235Z");
- doTest("check stored and docValues value is correct", "test_s_dv", "str", "storedAndDocValues");
- doTest("check non-stored and non-indexed is accessible", "test_s_dvo2", "str", "gotIt");
- doTest("enumField", "enum_dvo", "str", "Critical");
+ public void testRandomSingleAndMultiValued() throws Exception {
+ for (int c = 0 ; c < 10 * RANDOM_MULTIPLIER ; ++c) {
+ clearIndex();
+ int[] arity = new int[9];
+ for (int a = 0 ; a < arity.length ; ++a) {
+ // Single-valued 50% of the time; other 50%: 2-10 values equally likely
+ arity[a] = random().nextBoolean() ? 1 : TestUtil.nextInt(random(), 2, 10);
+ }
+ doTest("check string value is correct", dvStringFieldName(arity[0], true, false), "str", nextValues(arity[0], "str"));
+ doTest("check int value is correct", "test_i" + plural(arity[1]) + "_dvo", "int", nextValues(arity[1], "int"));
+ doTest("check double value is correct", "test_d" + plural(arity[2]) + "_dvo", "double", nextValues(arity[2], "double"));
+ doTest("check long value is correct", "test_l" + plural(arity[3]) + "_dvo", "long", nextValues(arity[3], "long"));
+ doTest("check float value is correct", "test_f" + plural(arity[4]) + "_dvo", "float", nextValues(arity[4], "float"));
+ doTest("check date value is correct", "test_dt" + plural(arity[5]) + "_dvo", "date", nextValues(arity[5], "date"));
+ doTest("check stored and docValues value is correct", dvStringFieldName(arity[6], true, true), "str", nextValues(arity[6], "str"));
+ doTest("check non-stored and non-indexed is accessible", dvStringFieldName(arity[7], false, false), "str", nextValues(arity[7], "str"));
+ doTest("enumField", "enum" + plural(arity[8]) + "_dvo", "str", nextValues(arity[8], "enum"));
+ }
}
- @Test
- public void testMultiValued() throws IOException {
- clearIndex();
- doTest("check string value is correct", "test_ss_dvo", "str", "keyword", "keyword2");
- doTest("check int value is correct", "test_is_dvo", "int", "1234", "12345");
- doTest("check double value is correct", "test_ds_dvo", "double", "1.234", "12.34", "123.4");
- doTest("check long value is correct", "test_ls_dvo", "long", "12345", "123456");
- doTest("check float value is correct", "test_fs_dvo", "float", "1.234", "12.34");
- doTest("check dt value is correct", "test_dts_dvo", "date", "1976-07-04T12:08:56.235Z", "1978-07-04T12:08:56.235Z");
- doTest("check stored and docValues value is correct", "test_ss_dv", "str", "storedAndDocValues", "storedAndDocValues2");
- doTest("check non-stored and non-indexed is accessible", "test_ss_dvo2", "str", "gotIt", "gotIt2");
- doTest("enumField", "enums_dvo", "str", "High", "Critical");
+ private String plural(int arity) {
+ return arity > 1 ? "s" : "";
+ }
+
+ private static boolean isStoredField(String fieldName) {
+ return STORED_FIELD_NAME_PATTERN.matcher(fieldName).find();
+ }
+
+ private String dvStringFieldName(int arity, boolean indexed, boolean stored) {
+ String base = "test_s" + (arity > 1 ? "s": "");
+ String suffix = "";
+ if (indexed && stored) suffix = "_dv";
+ else if (indexed && ! stored) suffix = "_dvo";
+ else if ( ! indexed && ! stored) suffix = "_dvo2";
+ else assertTrue("unsupported dv string field combination: stored and not indexed", false);
+ return base + suffix;
+ }
+
+ private String[] nextValues(int arity, String valueType) throws Exception {
+ String[] values = new String[arity];
+ for (int i = 0 ; i < arity ; ++i) {
+ switch (valueType) {
+ case "int": values[i] = String.valueOf(random().nextInt()); break;
+ case "double": values[i] = String.valueOf(Double.longBitsToDouble(random().nextLong())); break;
+ case "long": values[i] = String.valueOf(random().nextLong()); break;
+ case "float": values[i] = String.valueOf(Float.intBitsToFloat(random().nextInt())); break;
+ case "enum": values[i] = SEVERITY[TestUtil.nextInt(random(), 0, SEVERITY.length - 1)]; break;
+ case "str": {
+ String str = TestUtil.randomRealisticUnicodeString(random());
+ values[i] = BAD_CHAR_PATTERN.matcher(str).replaceAll("\uFFFD");
+ break;
+ }
+ case "date": {
+ long epochMillis = TestUtil.nextLong(random(), START_RANDOM_EPOCH_MILLIS, END_RANDOM_EPOCH_MILLIS);
+ LocalDateTime dateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMillis), ZoneOffset.UTC);
+ values[i] = dateTime.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME) + 'Z';
+ break;
+ }
+ default: throw new Exception("unknown type '" + valueType + "'");
+ }
+ }
+ return values;
}
@Test
@@ -198,6 +282,8 @@ public class TestUseDocValuesAsStored extends AbstractBadConfigTestBase {
String[] xpaths = new String[value.length + 1];
if (value.length > 1) {
+ Set<String> valueSet = new HashSet<>();
+ valueSet.addAll(Arrays.asList(value));
String[] fieldAndValues = new String[value.length * 2 + 2];
fieldAndValues[0] = "id";
fieldAndValues[1] = id;
@@ -205,10 +291,12 @@ public class TestUseDocValuesAsStored extends AbstractBadConfigTestBase {
for (int i = 0; i < value.length; ++i) {
fieldAndValues[i * 2 + 2] = field;
fieldAndValues[i * 2 + 3] = value[i];
- xpaths[i] = "//arr[@name='" + field + "']/" + type + "[" + (i + 1) + "][.='" + value[i] + "']";
+ xpaths[i] = "//arr[@name='" + field + "']/" + type + "[.='" + value[i] + "']";
}
- xpaths[value.length] = "*[count(//arr[@name='" + field + "']/" + type + ") = " + value.length + "]";
+ // Docvalues are sets, but stored values are ordered multisets, so cardinality depends on the value source
+ xpaths[value.length] = "*[count(//arr[@name='" + field + "']/" + type + ") = "
+ + (isStoredField(field) ? value.length : valueSet.size()) + "]";
assertU(adoc(fieldAndValues));
} else {
[02/50] lucene-solr:jira/SOLR-445: LUCENE-7098: reduce OfflineSorter
and BKDWriter IO by using 4 bytes instead of 8 bytes to encord ord in the
common case
Posted by ho...@apache.org.
LUCENE-7098: reduce OfflineSorter and BKDWriter IO by using 4 bytes instead of 8 bytes to encord ord in the common case
Squashed commit of the following:
commit 5ac2dcf2a972e46ccda3e7a8d8df5d0af58f712a
Merge: 68acf7f 684b222
Author: Mike McCandless <mi...@apache.org>
Date: Fri Mar 11 19:04:13 2016 -0500
Merge branch 'master' into intords
commit 68acf7f9ee2e0249d90075bc035721c0a91619f7
Author: Mike McCandless <mi...@apache.org>
Date: Fri Mar 11 19:04:01 2016 -0500
rename to totalPointCount and add comment; enforce that the caller doesn't exceed what they said; simplify the longOrds check to just compare to Integer.MAX_VALUE
commit afc964b56015475e8c354fdc8b0e05c7fa074ec2
Merge: db79e36 fe21f7a
Author: Mike McCandless <mi...@apache.org>
Date: Fri Mar 11 10:17:09 2016 -0500
Merge branch 'master' into intords
Conflicts:
lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
commit db79e365e097153a05813eaa70603c601bce1853
Author: Mike McCandless <mi...@apache.org>
Date: Fri Mar 11 10:15:05 2016 -0500
use int (4 bytes) not lon (8 bytes) if the number of points is less than ~2.1B
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b8cfcaf3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b8cfcaf3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b8cfcaf3
Branch: refs/heads/jira/SOLR-445
Commit: b8cfcaf3447004447ae6b600ab4ae7dd5baf69e3
Parents: 459d6fd
Author: Mike McCandless <mi...@apache.org>
Authored: Sat Mar 12 05:11:51 2016 -0500
Committer: Mike McCandless <mi...@apache.org>
Committed: Sat Mar 12 05:12:46 2016 -0500
----------------------------------------------------------------------
.../simpletext/SimpleTextPointsWriter.java | 3 +-
.../org/apache/lucene/codecs/PointsWriter.java | 14 +++-
.../codecs/lucene60/Lucene60PointsWriter.java | 20 ++++-
.../apache/lucene/index/PointValuesWriter.java | 16 ++--
.../org/apache/lucene/util/bkd/BKDWriter.java | 83 +++++++++++++-------
.../apache/lucene/util/bkd/HeapPointReader.java | 12 ++-
.../apache/lucene/util/bkd/HeapPointWriter.java | 37 +++++++--
.../lucene/util/bkd/OfflinePointReader.java | 24 ++++--
.../lucene/util/bkd/OfflinePointWriter.java | 17 +++-
.../org/apache/lucene/util/bkd/TestBKD.java | 22 ++++--
10 files changed, 178 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8cfcaf3/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java
index d2b848d..13494f5 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java
@@ -76,7 +76,8 @@ class SimpleTextPointsWriter extends PointsWriter {
fieldInfo.getPointDimensionCount(),
fieldInfo.getPointNumBytes(),
BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE,
- BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP) {
+ BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP,
+ values.size(fieldInfo.name)) {
@Override
protected void writeIndex(IndexOutput out, long[] leafBlockFPs, byte[] splitPackedValues) throws IOException {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8cfcaf3/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
index 53db281..000d713 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
@@ -41,6 +41,17 @@ public abstract class PointsWriter implements Closeable {
* from the incoming segment. The default codec overrides this for 1D fields and uses
* a faster but more complex implementation. */
protected void mergeOneField(MergeState mergeState, FieldInfo fieldInfo) throws IOException {
+ long maxPointCount = 0;
+ for (int i=0;i<mergeState.pointsReaders.length;i++) {
+ PointsReader pointsReader = mergeState.pointsReaders[i];
+ if (pointsReader != null) {
+ FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(fieldInfo.name);
+ if (readerFieldInfo != null) {
+ maxPointCount += pointsReader.size(fieldInfo.name);
+ }
+ }
+ }
+ final long finalMaxPointCount = maxPointCount;
writeField(fieldInfo,
new PointsReader() {
@Override
@@ -48,6 +59,7 @@ public abstract class PointsWriter implements Closeable {
if (fieldName.equals(fieldInfo.name) == false) {
throw new IllegalArgumentException("field name must match the field being merged");
}
+
for (int i=0;i<mergeState.pointsReaders.length;i++) {
PointsReader pointsReader = mergeState.pointsReaders[i];
if (pointsReader == null) {
@@ -124,7 +136,7 @@ public abstract class PointsWriter implements Closeable {
@Override
public long size(String fieldName) {
- throw new UnsupportedOperationException();
+ return finalMaxPointCount;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8cfcaf3/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java
index 3d09c45..1148fb0 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java
@@ -88,7 +88,8 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
fieldInfo.getPointDimensionCount(),
fieldInfo.getPointNumBytes(),
maxPointsInLeafNode,
- maxMBSortInHeap)) {
+ maxMBSortInHeap,
+ values.size(fieldInfo.name))) {
values.intersect(fieldInfo.name, new IntersectVisitor() {
@Override
@@ -131,6 +132,20 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
for (FieldInfo fieldInfo : mergeState.mergeFieldInfos) {
if (fieldInfo.getPointDimensionCount() != 0) {
if (fieldInfo.getPointDimensionCount() == 1) {
+
+ // Worst case total maximum size (if none of the points are deleted):
+ long totMaxSize = 0;
+ for(int i=0;i<mergeState.pointsReaders.length;i++) {
+ PointsReader reader = mergeState.pointsReaders[i];
+ if (reader != null) {
+ FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
+ FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
+ if (readerFieldInfo != null) {
+ totMaxSize += reader.size(fieldInfo.name);
+ }
+ }
+ }
+
//System.out.println("MERGE: field=" + fieldInfo.name);
// Optimize the 1D case to use BKDWriter.merge, which does a single merge sort of the
// already sorted incoming segments, instead of trying to sort all points again as if
@@ -141,7 +156,8 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
fieldInfo.getPointDimensionCount(),
fieldInfo.getPointNumBytes(),
maxPointsInLeafNode,
- maxMBSortInHeap)) {
+ maxMBSortInHeap,
+ totMaxSize)) {
List<BKDReader> bkdReaders = new ArrayList<>();
List<MergeState.DocMap> docMaps = new ArrayList<>();
List<Integer> docIDBases = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8cfcaf3/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
index 35b9a90..694ccf6 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
@@ -31,7 +31,7 @@ class PointValuesWriter {
private final ByteBlockPool bytes;
private final Counter iwBytesUsed;
private int[] docIDs;
- private int numDocs;
+ private int numPoints;
private final byte[] packedValue;
public PointValuesWriter(DocumentsWriterPerThread docWriter, FieldInfo fieldInfo) {
@@ -51,13 +51,13 @@ class PointValuesWriter {
if (value.length != fieldInfo.getPointDimensionCount() * fieldInfo.getPointNumBytes()) {
throw new IllegalArgumentException("field=" + fieldInfo.name + ": this field's value has length=" + value.length + " but should be " + (fieldInfo.getPointDimensionCount() * fieldInfo.getPointNumBytes()));
}
- if (docIDs.length == numDocs) {
- docIDs = ArrayUtil.grow(docIDs, numDocs+1);
- iwBytesUsed.addAndGet((docIDs.length - numDocs) * Integer.BYTES);
+ if (docIDs.length == numPoints) {
+ docIDs = ArrayUtil.grow(docIDs, numPoints+1);
+ iwBytesUsed.addAndGet((docIDs.length - numPoints) * Integer.BYTES);
}
bytes.append(value);
- docIDs[numDocs] = docID;
- numDocs++;
+ docIDs[numPoints] = docID;
+ numPoints++;
}
public void flush(SegmentWriteState state, PointsWriter writer) throws IOException {
@@ -69,7 +69,7 @@ class PointValuesWriter {
if (fieldName.equals(fieldInfo.name) == false) {
throw new IllegalArgumentException("fieldName must be the same");
}
- for(int i=0;i<numDocs;i++) {
+ for(int i=0;i<numPoints;i++) {
bytes.readBytes(packedValue.length * i, packedValue, 0, packedValue.length);
visitor.visit(docIDs[i], packedValue);
}
@@ -111,7 +111,7 @@ class PointValuesWriter {
@Override
public long size(String fieldName) {
- throw new UnsupportedOperationException();
+ return numPoints;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8cfcaf3/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index f1aba9d..bb2402b 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -128,12 +128,18 @@ public class BKDWriter implements Closeable {
protected long pointCount;
- public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim) throws IOException {
- this(maxDoc, tempDir, tempFileNamePrefix, numDims, bytesPerDim, DEFAULT_MAX_POINTS_IN_LEAF_NODE, DEFAULT_MAX_MB_SORT_IN_HEAP);
+ /** true if we have so many values that we must write ords using long (8 bytes) instead of int (4 bytes) */
+ private final boolean longOrds;
+
+ /** An upper bound on how many points the caller will add (includes deletions) */
+ private final long totalPointCount;
+
+ public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim, long totalPointCount) throws IOException {
+ this(maxDoc, tempDir, tempFileNamePrefix, numDims, bytesPerDim, DEFAULT_MAX_POINTS_IN_LEAF_NODE, DEFAULT_MAX_MB_SORT_IN_HEAP, totalPointCount);
}
- public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim, int maxPointsInLeafNode, double maxMBSortInHeap) throws IOException {
- verifyParams(numDims, maxPointsInLeafNode, maxMBSortInHeap);
+ public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount) throws IOException {
+ verifyParams(numDims, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount);
// We use tracking dir to deal with removing files on exception, so each place that
// creates temp files doesn't need crazy try/finally/sucess logic:
this.tempDir = new TrackingDirectoryWrapper(tempDir);
@@ -141,6 +147,7 @@ public class BKDWriter implements Closeable {
this.maxPointsInLeafNode = maxPointsInLeafNode;
this.numDims = numDims;
this.bytesPerDim = bytesPerDim;
+ this.totalPointCount = totalPointCount;
docsSeen = new FixedBitSet(maxDoc);
packedBytesLength = numDims * bytesPerDim;
@@ -153,8 +160,15 @@ public class BKDWriter implements Closeable {
minPackedValue = new byte[packedBytesLength];
maxPackedValue = new byte[packedBytesLength];
- // dimensional values (numDims * bytesPerDim) + ord (long) + docID (int)
- bytesPerDoc = packedBytesLength + Long.BYTES + Integer.BYTES;
+ // If we may have more than 1+Integer.MAX_VALUE values, then we must encode ords with long (8 bytes), else we can use int (4 bytes).
+ longOrds = totalPointCount > Integer.MAX_VALUE;
+
+ // dimensional values (numDims * bytesPerDim) + ord (int or long) + docID (int)
+ if (longOrds) {
+ bytesPerDoc = packedBytesLength + Long.BYTES + Integer.BYTES;
+ } else {
+ bytesPerDoc = packedBytesLength + Integer.BYTES + Integer.BYTES;
+ }
// As we recurse, we compute temporary partitions of the data, halving the
// number of points at each recursion. Once there are few enough points,
@@ -173,12 +187,12 @@ public class BKDWriter implements Closeable {
}
// We write first maxPointsSortInHeap in heap, then cutover to offline for additional points:
- heapPointWriter = new HeapPointWriter(16, maxPointsSortInHeap, packedBytesLength);
+ heapPointWriter = new HeapPointWriter(16, maxPointsSortInHeap, packedBytesLength, longOrds);
this.maxMBSortInHeap = maxMBSortInHeap;
}
- public static void verifyParams(int numDims, int maxPointsInLeafNode, double maxMBSortInHeap) {
+ public static void verifyParams(int numDims, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount) {
// We encode dim in a single byte in the splitPackedValues, but we only expose 4 bits for it now, in case we want to use
// remaining 4 bits for another purpose later
if (numDims < 1 || numDims > MAX_DIMS) {
@@ -193,13 +207,16 @@ public class BKDWriter implements Closeable {
if (maxMBSortInHeap < 0.0) {
throw new IllegalArgumentException("maxMBSortInHeap must be >= 0.0 (got: " + maxMBSortInHeap + ")");
}
+ if (totalPointCount < 0) {
+ throw new IllegalArgumentException("totalPointCount must be >=0 (got: " + totalPointCount + ")");
+ }
}
/** If the current segment has too many points then we switchover to temp files / offline sort. */
private void switchToOffline() throws IOException {
// For each .add we just append to this input file, then in .finish we sort this input and resursively build the tree:
- offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength);
+ offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds);
tempInput = offlinePointWriter.out;
PointReader reader = heapPointWriter.getReader(0);
for(int i=0;i<pointCount;i++) {
@@ -243,6 +260,9 @@ public class BKDWriter implements Closeable {
}
pointCount++;
+ if (pointCount > totalPointCount) {
+ throw new IllegalStateException("totalPointCount=" + totalPointCount + " was passed when we were created, but we just hit " + pointCount + " values");
+ }
docsSeen.set(docID);
}
@@ -437,6 +457,9 @@ public class BKDWriter implements Closeable {
assert numDims > 1 || valueInOrder(valueCount, lastPackedValue, reader.state.scratchPackedValue);
valueCount++;
+ if (pointCount > totalPointCount) {
+ throw new IllegalStateException("totalPointCount=" + totalPointCount + " was passed when we were created, but we just hit " + pointCount + " values");
+ }
if (leafCount == 0) {
if (leafBlockFPs.size() > 0) {
@@ -569,13 +592,10 @@ public class BKDWriter implements Closeable {
new IntroSorter() {
private final byte[] pivotPackedValue = new byte[bytesPerDim];
private int pivotDocID;
- private long pivotOrd;
@Override
protected void setPivot(int i) {
pivotDocID = writer.docIDs[i];
- pivotOrd = writer.ords[i];
-
int block = i / writer.valuesPerBlock;
int index = i % writer.valuesPerBlock;
System.arraycopy(writer.blocks.get(block), index*packedBytesLength+dim*bytesPerDim, pivotPackedValue, 0, bytesPerDim);
@@ -593,12 +613,7 @@ public class BKDWriter implements Closeable {
}
// Tie-break
- cmp = Integer.compare(pivotDocID, writer.docIDs[j]);
- if (cmp != 0) {
- return cmp;
- }
-
- return Long.compare(pivotOrd, writer.ords[j]);
+ return Integer.compare(pivotDocID, writer.docIDs[j]);
}
@Override
@@ -607,9 +622,15 @@ public class BKDWriter implements Closeable {
writer.docIDs[i] = writer.docIDs[j];
writer.docIDs[j] = docID;
- long ord = writer.ords[i];
- writer.ords[i] = writer.ords[j];
- writer.ords[j] = ord;
+ if (longOrds) {
+ long ord = writer.ordsLong[i];
+ writer.ordsLong[i] = writer.ordsLong[j];
+ writer.ordsLong[j] = ord;
+ } else {
+ int ord = writer.ords[i];
+ writer.ords[i] = writer.ords[j];
+ writer.ords[j] = ord;
+ }
byte[] blockI = writer.blocks.get(i / writer.valuesPerBlock);
int indexI = (i % writer.valuesPerBlock) * packedBytesLength;
@@ -660,7 +681,7 @@ public class BKDWriter implements Closeable {
sorted = heapPointWriter;
} else {
// Subsequent dims need a private copy
- sorted = new HeapPointWriter((int) pointCount, (int) pointCount, packedBytesLength);
+ sorted = new HeapPointWriter((int) pointCount, (int) pointCount, packedBytesLength, longOrds);
sorted.copyFrom(heapPointWriter);
}
@@ -691,10 +712,16 @@ public class BKDWriter implements Closeable {
}
// Tie-break by docID:
- reader.reset(a.bytes, a.offset + packedBytesLength + Long.BYTES, a.length);
+ int offset;
+ if (longOrds) {
+ offset = Long.BYTES;
+ } else {
+ offset = Integer.BYTES;
+ }
+ reader.reset(a.bytes, a.offset + packedBytesLength + offset, a.length);
final int docIDA = reader.readInt();
- reader.reset(b.bytes, b.offset + packedBytesLength + Long.BYTES, b.length);
+ reader.reset(b.bytes, b.offset + packedBytesLength + offset, b.length);
final int docIDB = reader.readInt();
// No need to tie break on ord, for the case where the same doc has the same value in a given dimension indexed more than once: it
@@ -746,7 +773,7 @@ public class BKDWriter implements Closeable {
assert lastWriter[0] != null;
- return new OfflinePointWriter(tempDir, lastWriter[0], packedBytesLength, pointCount);
+ return new OfflinePointWriter(tempDir, lastWriter[0], packedBytesLength, pointCount, longOrds);
}
}
@@ -1005,7 +1032,7 @@ public class BKDWriter implements Closeable {
private PathSlice switchToHeap(PathSlice source) throws IOException {
int count = Math.toIntExact(source.count);
try (
- PointWriter writer = new HeapPointWriter(count, count, packedBytesLength);
+ PointWriter writer = new HeapPointWriter(count, count, packedBytesLength, longOrds);
PointReader reader = source.writer.getReader(source.start);
) {
for(int i=0;i<count;i++) {
@@ -1214,9 +1241,9 @@ public class BKDWriter implements Closeable {
PointWriter getPointWriter(long count) throws IOException {
if (count <= maxPointsSortInHeap) {
int size = Math.toIntExact(count);
- return new HeapPointWriter(size, size, packedBytesLength);
+ return new HeapPointWriter(size, size, packedBytesLength, longOrds);
} else {
- return new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength);
+ return new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8cfcaf3/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
index 081360b..b178f08 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
@@ -26,15 +26,17 @@ final class HeapPointReader implements PointReader {
final List<byte[]> blocks;
final int valuesPerBlock;
final int packedBytesLength;
- final long[] ords;
+ final long[] ordsLong;
+ final int[] ords;
final int[] docIDs;
final int end;
final byte[] scratch;
- HeapPointReader(List<byte[]> blocks, int valuesPerBlock, int packedBytesLength, long[] ords, int[] docIDs, int start, int end) {
+ HeapPointReader(List<byte[]> blocks, int valuesPerBlock, int packedBytesLength, int[] ords, long[] ordsLong, int[] docIDs, int start, int end) {
this.blocks = blocks;
this.valuesPerBlock = valuesPerBlock;
this.ords = ords;
+ this.ordsLong = ordsLong;
this.docIDs = docIDs;
curRead = start-1;
this.end = end;
@@ -76,7 +78,11 @@ final class HeapPointReader implements PointReader {
@Override
public long ord() {
- return ords[curRead];
+ if (ordsLong != null) {
+ return ordsLong[curRead];
+ } else {
+ return ords[curRead];
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8cfcaf3/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
index 0236187..3b043d0 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
@@ -23,7 +23,8 @@ import org.apache.lucene.util.ArrayUtil;
final class HeapPointWriter implements PointWriter {
int[] docIDs;
- long[] ords;
+ long[] ordsLong;
+ int[] ords;
private int nextWrite;
private boolean closed;
final int maxSize;
@@ -32,11 +33,15 @@ final class HeapPointWriter implements PointWriter {
// NOTE: can't use ByteBlockPool because we need random-write access when sorting in heap
final List<byte[]> blocks = new ArrayList<>();
- public HeapPointWriter(int initSize, int maxSize, int packedBytesLength) {
+ public HeapPointWriter(int initSize, int maxSize, int packedBytesLength, boolean longOrds) {
docIDs = new int[initSize];
- ords = new long[initSize];
this.maxSize = maxSize;
this.packedBytesLength = packedBytesLength;
+ if (longOrds) {
+ this.ordsLong = new long[initSize];
+ } else {
+ this.ords = new int[initSize];
+ }
// 4K per page, unless each value is > 4K:
valuesPerBlock = Math.max(1, 4096/packedBytesLength);
}
@@ -46,7 +51,14 @@ final class HeapPointWriter implements PointWriter {
throw new IllegalStateException("docIDs.length=" + docIDs.length + " other.nextWrite=" + other.nextWrite);
}
System.arraycopy(other.docIDs, 0, docIDs, 0, other.nextWrite);
- System.arraycopy(other.ords, 0, ords, 0, other.nextWrite);
+ if (other.ords != null) {
+ assert this.ords != null;
+ System.arraycopy(other.ords, 0, ords, 0, other.nextWrite);
+ } else {
+ assert this.ordsLong != null;
+ System.arraycopy(other.ordsLong, 0, ordsLong, 0, other.nextWrite);
+ }
+
for(byte[] block : other.blocks) {
blocks.add(block.clone());
}
@@ -91,21 +103,30 @@ final class HeapPointWriter implements PointWriter {
public void append(byte[] packedValue, long ord, int docID) {
assert closed == false;
assert packedValue.length == packedBytesLength;
- if (ords.length == nextWrite) {
+ if (docIDs.length == nextWrite) {
int nextSize = Math.min(maxSize, ArrayUtil.oversize(nextWrite+1, Integer.BYTES));
assert nextSize > nextWrite: "nextSize=" + nextSize + " vs nextWrite=" + nextWrite;
- ords = growExact(ords, nextSize);
docIDs = growExact(docIDs, nextSize);
+ if (ordsLong != null) {
+ ordsLong = growExact(ordsLong, nextSize);
+ } else {
+ ords = growExact(ords, nextSize);
+ }
}
writePackedValue(nextWrite, packedValue);
- ords[nextWrite] = ord;
+ if (ordsLong != null) {
+ ordsLong[nextWrite] = ord;
+ } else {
+ assert ord <= Integer.MAX_VALUE;
+ ords[nextWrite] = (int) ord;
+ }
docIDs[nextWrite] = docID;
nextWrite++;
}
@Override
public PointReader getReader(long start) {
- return new HeapPointReader(blocks, valuesPerBlock, packedBytesLength, ords, docIDs, (int) start, nextWrite);
+ return new HeapPointReader(blocks, valuesPerBlock, packedBytesLength, ords, ordsLong, docIDs, (int) start, nextWrite);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8cfcaf3/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java
index 83d863b..3c4b8b5 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java
@@ -30,18 +30,22 @@ final class OfflinePointReader implements PointReader {
private final byte[] packedValue;
private long ord;
private int docID;
+ // true if ords are written as long (8 bytes), else 4 bytes
+ private boolean longOrds;
- OfflinePointReader(Directory tempDir, String tempFileName, int packedBytesLength, long start, long length) throws IOException {
- this(tempDir.openInput(tempFileName, IOContext.READONCE), packedBytesLength, start, length);
- }
-
- private OfflinePointReader(IndexInput in, int packedBytesLength, long start, long length) throws IOException {
- this.in = in;
- int bytesPerDoc = packedBytesLength + Long.BYTES + Integer.BYTES;
+ OfflinePointReader(Directory tempDir, String tempFileName, int packedBytesLength, long start, long length, boolean longOrds) throws IOException {
+ in = tempDir.openInput(tempFileName, IOContext.READONCE);
+ int bytesPerDoc = packedBytesLength + Integer.BYTES;
+ if (longOrds) {
+ bytesPerDoc += Long.BYTES;
+ } else {
+ bytesPerDoc += Integer.BYTES;
+ }
long seekFP = start * bytesPerDoc;
in.seek(seekFP);
this.countLeft = length;
packedValue = new byte[packedBytesLength];
+ this.longOrds = longOrds;
}
@Override
@@ -58,7 +62,11 @@ final class OfflinePointReader implements PointReader {
assert countLeft == -1;
return false;
}
- ord = in.readLong();
+ if (longOrds) {
+ ord = in.readLong();
+ } else {
+ ord = in.readInt();
+ }
docID = in.readInt();
return true;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8cfcaf3/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
index 625e6fa..dcf6781 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
@@ -30,27 +30,36 @@ final class OfflinePointWriter implements PointWriter {
final int packedBytesLength;
private long count;
private boolean closed;
+ // true if ords are written as long (8 bytes), else 4 bytes
+ private boolean longOrds;
- public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int packedBytesLength) throws IOException {
+ public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int packedBytesLength, boolean longOrds) throws IOException {
this.out = tempDir.createTempOutput(tempFileNamePrefix, "bkd", IOContext.DEFAULT);
this.tempDir = tempDir;
this.packedBytesLength = packedBytesLength;
+ this.longOrds = longOrds;
}
/** Initializes on an already written/closed file, just so consumers can use {@link #getReader} to read the file. */
- public OfflinePointWriter(Directory tempDir, IndexOutput out, int packedBytesLength, long count) {
+ public OfflinePointWriter(Directory tempDir, IndexOutput out, int packedBytesLength, long count, boolean longOrds) {
this.out = out;
this.tempDir = tempDir;
this.packedBytesLength = packedBytesLength;
this.count = count;
closed = true;
+ this.longOrds = longOrds;
}
@Override
public void append(byte[] packedValue, long ord, int docID) throws IOException {
assert packedValue.length == packedBytesLength;
out.writeBytes(packedValue, 0, packedValue.length);
- out.writeLong(ord);
+ if (longOrds) {
+ out.writeLong(ord);
+ } else {
+ assert ord <= Integer.MAX_VALUE;
+ out.writeInt((int) ord);
+ }
out.writeInt(docID);
count++;
}
@@ -58,7 +67,7 @@ final class OfflinePointWriter implements PointWriter {
@Override
public PointReader getReader(long start) throws IOException {
assert closed;
- return new OfflinePointReader(tempDir, out.getName(), packedBytesLength, start, count-start);
+ return new OfflinePointReader(tempDir, out.getName(), packedBytesLength, start, count-start, longOrds);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8cfcaf3/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
index f1402fc..2017743 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
@@ -40,9 +40,17 @@ import org.apache.lucene.util.TestUtil;
public class TestBKD extends LuceneTestCase {
+ private long randomPointCount() {
+ if (random().nextBoolean()) {
+ return random().nextInt(Integer.MAX_VALUE);
+ } else {
+ return random().nextLong() & Long.MAX_VALUE;
+ }
+ }
+
public void testBasicInts1D() throws Exception {
try (Directory dir = getDirectory(100)) {
- BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 4, 2, 1.0f);
+ BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 4, 2, 1.0f, randomPointCount());
byte[] scratch = new byte[4];
for(int docID=0;docID<100;docID++) {
NumericUtils.intToSortableBytes(docID, scratch, 0);
@@ -117,7 +125,7 @@ public class TestBKD extends LuceneTestCase {
int numDims = TestUtil.nextInt(random(), 1, 5);
int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100);
float maxMB = (float) 3.0 + (3*random().nextFloat());
- BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, 4, maxPointsInLeafNode, maxMB);
+ BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, 4, maxPointsInLeafNode, maxMB, randomPointCount());
if (VERBOSE) {
System.out.println("TEST: numDims=" + numDims + " numDocs=" + numDocs);
@@ -258,7 +266,7 @@ public class TestBKD extends LuceneTestCase {
int numDims = TestUtil.nextInt(random(), 1, 5);
int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100);
float maxMB = (float) 3.0 + (3*random().nextFloat());
- BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
+ BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount());
BigInteger[][] docs = new BigInteger[numDocs][];
byte[] scratch = new byte[numBytesPerDim*numDims];
@@ -431,7 +439,7 @@ public class TestBKD extends LuceneTestCase {
public void testTooLittleHeap() throws Exception {
try (Directory dir = getDirectory(0)) {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
- new BKDWriter(1, dir, "bkd", 1, 16, 1000000, 0.001);
+ new BKDWriter(1, dir, "bkd", 1, 16, 1000000, 0.001, randomPointCount());
});
assertTrue(expected.getMessage().contains("either increase maxMBSortInHeap or decrease maxPointsInLeafNode"));
}
@@ -554,7 +562,7 @@ public class TestBKD extends LuceneTestCase {
List<Integer> docIDBases = null;
int seg = 0;
- BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
+ BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount());
IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT);
IndexInput in = null;
@@ -608,7 +616,7 @@ public class TestBKD extends LuceneTestCase {
seg++;
maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 1000);
maxMB = (float) 3.0 + (3*random().nextDouble());
- w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
+ w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount());
lastDocIDBase = docID;
}
}
@@ -623,7 +631,7 @@ public class TestBKD extends LuceneTestCase {
out.close();
in = dir.openInput("bkd", IOContext.DEFAULT);
seg++;
- w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
+ w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount());
List<BKDReader> readers = new ArrayList<>();
for(long fp : toMerge) {
in.seek(fp);
[23/50] lucene-solr:jira/SOLR-445: LUCENE-7091: Added doc values
support to memory index
Posted by ho...@apache.org.
LUCENE-7091: Added doc values support to memory index
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cf3eea26
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cf3eea26
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cf3eea26
Branch: refs/heads/jira/SOLR-445
Commit: cf3eea26406692306505d2606d7ff73ee3634c30
Parents: 8185c8a
Author: Martijn van Groningen <ma...@gmail.com>
Authored: Wed Mar 9 17:38:23 2016 +0100
Committer: Martijn van Groningen <mv...@apache.org>
Committed: Tue Mar 15 09:57:40 2016 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../apache/lucene/index/memory/MemoryIndex.java | 465 +++++++++++++++----
.../lucene/index/memory/TestMemoryIndex.java | 133 ++++++
.../memory/TestMemoryIndexAgainstRAMDir.java | 134 ++++++
4 files changed, 636 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf3eea26/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 10d4d10..db08eb3 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -203,6 +203,9 @@ Other
* LUCENE-7087: Let MemoryIndex#fromDocument(...) accept 'Iterable<? extends IndexableField>'
as document instead of 'Document'. (Martijn van Groningen)
+* LUCENE-7091: Add doc values support to MemoryIndex
+ (Martijn van Groningen, David Smiley)
+
======================= Lucene 5.5.0 =======================
New Features
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf3eea26/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index 9e01182..40159aa 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -17,12 +17,15 @@
package org.apache.lucene.index.memory;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
+import java.util.Locale;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.stream.Collectors;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
@@ -255,7 +258,7 @@ public class MemoryIndex {
throw new IllegalArgumentException("analyzer must not be null");
TokenStream stream = analyzer.tokenStream(fieldName, text);
- addField(fieldName, stream, 1.0f, analyzer.getPositionIncrementGap(fieldName), analyzer.getOffsetGap(fieldName));
+ addField(fieldName, stream, 1.0f, analyzer.getPositionIncrementGap(fieldName), analyzer.getOffsetGap(fieldName), DocValuesType.NONE, null);
}
/**
@@ -351,7 +354,9 @@ public class MemoryIndex {
}
/**
- * Adds a lucene {@link IndexableField} to the MemoryIndex using the provided analyzer
+ * Adds a lucene {@link IndexableField} to the MemoryIndex using the provided analyzer.
+ * Also stores doc values based on {@link IndexableFieldType#docValuesType()} if set.
+ *
* @param field the field to add
* @param analyzer the analyzer to use for term analysis
* @throws IllegalArgumentException if the field is a DocValues or Point field, as these
@@ -362,7 +367,9 @@ public class MemoryIndex {
}
/**
- * Adds a lucene {@link IndexableField} to the MemoryIndex using the provided analyzer
+ * Adds a lucene {@link IndexableField} to the MemoryIndex using the provided analyzer.
+ * Also stores doc values based on {@link IndexableFieldType#docValuesType()} if set.
+ *
* @param field the field to add
* @param analyzer the analyzer to use for term analysis
* @param boost a field boost
@@ -370,17 +377,42 @@ public class MemoryIndex {
* structures are not supported by MemoryIndex
*/
public void addField(IndexableField field, Analyzer analyzer, float boost) {
- if (field.fieldType().docValuesType() != DocValuesType.NONE)
- throw new IllegalArgumentException("MemoryIndex does not support DocValues fields");
- if (field.fieldType().pointDimensionCount() != 0)
+ if (field.fieldType().pointDimensionCount() != 0) {
throw new IllegalArgumentException("MemoryIndex does not support Points");
- if (analyzer == null) {
- addField(field.name(), field.tokenStream(null, null), boost);
}
- else {
- addField(field.name(), field.tokenStream(analyzer, null), boost,
- analyzer.getPositionIncrementGap(field.name()), analyzer.getOffsetGap(field.name()));
+
+ int offsetGap;
+ TokenStream tokenStream;
+ int positionIncrementGap;
+ if (analyzer != null) {
+ offsetGap = analyzer.getOffsetGap(field.name());
+ tokenStream = field.tokenStream(analyzer, null);
+ positionIncrementGap = analyzer.getPositionIncrementGap(field.name());
+ } else {
+ offsetGap = 1;
+ tokenStream = field.tokenStream(null, null);
+ positionIncrementGap = 0;
+ }
+
+ DocValuesType docValuesType = field.fieldType().docValuesType();
+ Object docValuesValue;
+ switch (docValuesType) {
+ case NONE:
+ docValuesValue = null;
+ break;
+ case BINARY:
+ case SORTED:
+ case SORTED_SET:
+ docValuesValue = field.binaryValue();
+ break;
+ case NUMERIC:
+ case SORTED_NUMERIC:
+ docValuesValue = field.numericValue();
+ break;
+ default:
+ throw new UnsupportedOperationException("unknown doc values type [" + docValuesType + "]");
}
+ addField(field.name(), tokenStream, boost, positionIncrementGap, offsetGap, docValuesType, docValuesValue);
}
/**
@@ -448,66 +480,126 @@ public class MemoryIndex {
* the offset gap if fields with the same name are added more than once
* @see org.apache.lucene.document.Field#setBoost(float)
*/
- public void addField(String fieldName, TokenStream tokenStream, float boost, int positionIncrementGap,
- int offsetGap) {
- try (TokenStream stream = tokenStream) {
- if (frozen)
- throw new IllegalArgumentException("Cannot call addField() when MemoryIndex is frozen");
- if (fieldName == null)
- throw new IllegalArgumentException("fieldName must not be null");
- if (stream == null)
- throw new IllegalArgumentException("token stream must not be null");
- if (boost <= 0.0f)
- throw new IllegalArgumentException("boost factor must be greater than 0.0");
- int numTokens = 0;
- int numOverlapTokens = 0;
- int pos = -1;
- final BytesRefHash terms;
- final SliceByteStartArray sliceArray;
- Info info;
- long sumTotalTermFreq = 0;
- int offset = 0;
- FieldInfo fieldInfo;
- if ((info = fields.get(fieldName)) != null) {
- fieldInfo = info.fieldInfo;
- numTokens = info.numTokens;
- numOverlapTokens = info.numOverlapTokens;
- pos = info.lastPosition + positionIncrementGap;
- offset = info.lastOffset + offsetGap;
- terms = info.terms;
- boost *= info.boost;
- sliceArray = info.sliceArray;
- sumTotalTermFreq = info.sumTotalTermFreq;
- } else {
- fieldInfo = new FieldInfo(fieldName, fields.size(), true, false, this.storePayloads,
- this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS,
- DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0);
- sliceArray = new SliceByteStartArray(BytesRefHash.DEFAULT_CAPACITY);
- terms = new BytesRefHash(byteBlockPool, BytesRefHash.DEFAULT_CAPACITY, sliceArray);
- }
+ public void addField(String fieldName, TokenStream tokenStream, float boost, int positionIncrementGap, int offsetGap) {
+ addField(fieldName, tokenStream, boost, positionIncrementGap, offsetGap, DocValuesType.NONE, null);
+ }
+
+ private void addField(String fieldName, TokenStream tokenStream, float boost, int positionIncrementGap, int offsetGap,
+ DocValuesType docValuesType, Object docValuesValue) {
+
+ if (frozen) {
+ throw new IllegalArgumentException("Cannot call addField() when MemoryIndex is frozen");
+ }
+ if (fieldName == null) {
+ throw new IllegalArgumentException("fieldName must not be null");
+ }
+ if (boost <= 0.0f) {
+ throw new IllegalArgumentException("boost factor must be greater than 0.0");
+ }
+
+ Info info = fields.get(fieldName);
+ if (info == null) {
+ IndexOptions indexOptions = storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+ FieldInfo fieldInfo = new FieldInfo(fieldName, fields.size(), true, false, storePayloads, indexOptions, docValuesType, -1, Collections.emptyMap(), 0, 0);
+ fields.put(fieldName, info = new Info(fieldInfo, byteBlockPool));
+ }
+
+ if (docValuesType != DocValuesType.NONE) {
+ storeDocValues(info, docValuesType, docValuesValue);
+ }
+ if (tokenStream != null) {
+ storeTerms(info, tokenStream, boost, positionIncrementGap, offsetGap);
+ }
+ }
+ private void storeDocValues(Info info, DocValuesType docValuesType, Object docValuesValue) {
+ String fieldName = info.fieldInfo.name;
+ DocValuesType existingDocValuesType = info.fieldInfo.getDocValuesType();
+ if (existingDocValuesType == DocValuesType.NONE) {
+ // first time we add doc values for this field:
+ info.fieldInfo = new FieldInfo(
+ info.fieldInfo.name, info.fieldInfo.number, info.fieldInfo.hasVectors(), info.fieldInfo.hasPayloads(),
+ info.fieldInfo.hasPayloads(), info.fieldInfo.getIndexOptions(), docValuesType, -1, info.fieldInfo.attributes(),
+ info.fieldInfo.getPointDimensionCount(), info.fieldInfo.getPointNumBytes()
+ );
+ } else if (existingDocValuesType != docValuesType) {
+ throw new IllegalArgumentException("Can't add [" + docValuesType + "] doc values field [" + fieldName + "], because [" + existingDocValuesType + "] doc values field already exists");
+ }
+ switch (docValuesType) {
+ case NUMERIC:
+ if (info.numericProducer.dvLongValues != null) {
+ throw new IllegalArgumentException("Only one value per field allowed for [" + docValuesType + "] doc values field [" + fieldName + "]");
+ }
+ info.numericProducer.dvLongValues = new long[]{(long) docValuesValue};
+ info.numericProducer.count++;
+ break;
+ case SORTED_NUMERIC:
+ if (info.numericProducer.dvLongValues == null) {
+ info.numericProducer.dvLongValues = new long[4];
+ }
+ info.numericProducer.dvLongValues = ArrayUtil.grow(info.numericProducer.dvLongValues, info.numericProducer.count + 1);
+ info.numericProducer.dvLongValues[info.numericProducer.count++] = (long) docValuesValue;
+ break;
+ case BINARY:
+ if (info.binaryProducer.dvBytesValuesSet != null) {
+ throw new IllegalArgumentException("Only one value per field allowed for [" + docValuesType + "] doc values field [" + fieldName + "]");
+ }
+ info.binaryProducer.dvBytesValuesSet = new BytesRefHash(byteBlockPool);
+ info.binaryProducer.dvBytesValuesSet.add((BytesRef) docValuesValue);
+ break;
+ case SORTED:
+ if (info.binaryProducer.dvBytesValuesSet != null) {
+ throw new IllegalArgumentException("Only one value per field allowed for [" + docValuesType + "] doc values field [" + fieldName + "]");
+ }
+ info.binaryProducer.dvBytesValuesSet = new BytesRefHash(byteBlockPool);
+ info.binaryProducer.dvBytesValuesSet.add((BytesRef) docValuesValue);
+ break;
+ case SORTED_SET:
+ if (info.binaryProducer.dvBytesValuesSet == null) {
+ info.binaryProducer.dvBytesValuesSet = new BytesRefHash(byteBlockPool);
+ }
+ info.binaryProducer.dvBytesValuesSet.add((BytesRef) docValuesValue);
+ break;
+ default:
+ throw new UnsupportedOperationException("unknown doc values type [" + docValuesType + "]");
+ }
+ }
+
+ private void storeTerms(Info info, TokenStream tokenStream, float boost, int positionIncrementGap, int offsetGap) {
+ int pos = -1;
+ int offset = 0;
+ if (info.numTokens == 0) {
+ info.boost = boost;
+ } else if (info.numTokens > 0) {
+ pos = info.lastPosition + positionIncrementGap;
+ offset = info.lastOffset + offsetGap;
+ info.boost *= boost;
+ }
+
+ try (TokenStream stream = tokenStream) {
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);
OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);
PayloadAttribute payloadAtt = storePayloads ? stream.addAttribute(PayloadAttribute.class) : null;
stream.reset();
-
+
while (stream.incrementToken()) {
// if (DEBUG) System.err.println("token='" + term + "'");
- numTokens++;
+ info.numTokens++;
final int posIncr = posIncrAttribute.getPositionIncrement();
- if (posIncr == 0)
- numOverlapTokens++;
+ if (posIncr == 0) {
+ info.numOverlapTokens++;
+ }
pos += posIncr;
- int ord = terms.add(termAtt.getBytesRef());
+ int ord = info.terms.add(termAtt.getBytesRef());
if (ord < 0) {
ord = (-ord) - 1;
- postingsWriter.reset(sliceArray.end[ord]);
+ postingsWriter.reset(info.sliceArray.end[ord]);
} else {
- sliceArray.start[ord] = postingsWriter.startNewSlice();
+ info.sliceArray.start[ord] = postingsWriter.startNewSlice();
}
- sliceArray.freq[ord]++;
- sumTotalTermFreq++;
+ info.sliceArray.freq[ord]++;
+ info.sumTotalTermFreq++;
postingsWriter.writeInt(pos);
if (storeOffsets) {
postingsWriter.writeInt(offsetAtt.startOffset() + offset);
@@ -523,13 +615,12 @@ public class MemoryIndex {
}
postingsWriter.writeInt(pIndex);
}
- sliceArray.end[ord] = postingsWriter.getCurrentOffset();
+ info.sliceArray.end[ord] = postingsWriter.getCurrentOffset();
}
stream.end();
-
- // ensure infos.numTokens > 0 invariant; needed for correct operation of terms()
- if (numTokens > 0) {
- fields.put(fieldName, new Info(fieldInfo, terms, sliceArray, numTokens, numOverlapTokens, boost, pos, offsetAtt.endOffset() + offset, sumTotalTermFreq));
+ if (info.numTokens > 0) {
+ info.lastPosition = pos;
+ info.lastOffset = offsetAtt.endOffset() + offset;
}
} catch (IOException e) {
throw new RuntimeException(e);
@@ -573,8 +664,7 @@ public class MemoryIndex {
public void freeze() {
this.frozen = true;
for (Info info : fields.values()) {
- info.sortTerms();
- info.getNormDocValues();//lazily computed
+ info.freeze();
}
}
@@ -702,7 +792,7 @@ public class MemoryIndex {
*/
private final class Info {
- private final FieldInfo fieldInfo;
+ private FieldInfo fieldInfo;
/** The norms for this field; computed on demand. */
private transient NumericDocValues norms;
@@ -711,40 +801,48 @@ public class MemoryIndex {
* Term strings and their positions for this field: Map <String
* termText, ArrayIntList positions>
*/
- private final BytesRefHash terms; // note unfortunate variable name class with Terms type
+ private BytesRefHash terms; // note unfortunate variable name class with Terms type
- private final SliceByteStartArray sliceArray;
+ private SliceByteStartArray sliceArray;
/** Terms sorted ascending by term text; computed on demand */
private transient int[] sortedTerms;
/** Number of added tokens for this field */
- private final int numTokens;
+ private int numTokens;
/** Number of overlapping tokens for this field */
- private final int numOverlapTokens;
+ private int numOverlapTokens;
/** Boost factor for hits for this field */
- private final float boost;
+ private float boost;
- private final long sumTotalTermFreq;
+ private long sumTotalTermFreq;
/** the last position encountered in this field for multi field support*/
- private final int lastPosition;
+ private int lastPosition;
/** the last offset encountered in this field for multi field support*/
- private final int lastOffset;
+ private int lastOffset;
- public Info(FieldInfo fieldInfo, BytesRefHash terms, SliceByteStartArray sliceArray, int numTokens, int numOverlapTokens, float boost, int lastPosition, int lastOffset, long sumTotalTermFreq) {
+ private BinaryDocValuesProducer binaryProducer;
+
+ private NumericDocValuesProducer numericProducer;
+
+ private boolean preparedDocValues;
+
+ private Info(FieldInfo fieldInfo, ByteBlockPool byteBlockPool) {
this.fieldInfo = fieldInfo;
- this.terms = terms;
- this.sliceArray = sliceArray;
- this.numTokens = numTokens;
- this.numOverlapTokens = numOverlapTokens;
- this.boost = boost;
- this.sumTotalTermFreq = sumTotalTermFreq;
- this.lastPosition = lastPosition;
- this.lastOffset = lastOffset;
+ this.sliceArray = new SliceByteStartArray(BytesRefHash.DEFAULT_CAPACITY);
+ this.terms = new BytesRefHash(byteBlockPool, BytesRefHash.DEFAULT_CAPACITY, sliceArray);;
+ this.binaryProducer = new BinaryDocValuesProducer();
+ this.numericProducer = new NumericDocValuesProducer();
+ }
+
+ void freeze() {
+ sortTerms();
+ prepareDocValues();
+ getNormDocValues();
}
/**
@@ -755,13 +853,26 @@ public class MemoryIndex {
* (which would be an alternative and somewhat more elegant approach,
* apart from more sophisticated Tries / prefix trees).
*/
- public void sortTerms() {
+ void sortTerms() {
if (sortedTerms == null) {
sortedTerms = terms.sort();
}
}
- public NumericDocValues getNormDocValues() {
+ void prepareDocValues() {
+ if (preparedDocValues == false) {
+ DocValuesType dvType = fieldInfo.getDocValuesType();
+ if (dvType == DocValuesType.NUMERIC || dvType == DocValuesType.SORTED_NUMERIC) {
+ numericProducer.prepareForUsage();
+ }
+ if (dvType == DocValuesType.BINARY || dvType == DocValuesType.SORTED || dvType == DocValuesType.SORTED_SET) {
+ binaryProducer.prepareForUsage();
+ }
+ preparedDocValues = true;
+ }
+ }
+
+ NumericDocValues getNormDocValues() {
if (norms == null) {
FieldInvertState invertState = new FieldInvertState(fieldInfo.name, fieldInfo.number,
numTokens, numOverlapTokens, 0, boost);
@@ -786,7 +897,81 @@ public class MemoryIndex {
///////////////////////////////////////////////////////////////////////////////
// Nested classes:
///////////////////////////////////////////////////////////////////////////////
-
+
+ private static final class BinaryDocValuesProducer {
+
+ BytesRefHash dvBytesValuesSet;
+ final SortedDocValues sortedDocValues;
+ final BytesRef spare = new BytesRef();
+
+ int[] bytesIds;
+
+ private BinaryDocValuesProducer() {
+ sortedDocValues = new SortedDocValues() {
+ @Override
+ public int getOrd(int docID) {
+ return 0;
+ }
+
+ @Override
+ public BytesRef lookupOrd(int ord) {
+ return getValue(ord);
+ }
+
+ @Override
+ public int getValueCount() {
+ return 1;
+ }
+ };
+ }
+
+ private void prepareForUsage() {
+ bytesIds = dvBytesValuesSet.sort();
+ }
+
+ private BytesRef getValue(int index) {
+ return dvBytesValuesSet.get(bytesIds[index], spare);
+ }
+
+ }
+
+ private static final class NumericDocValuesProducer {
+
+ long[] dvLongValues;
+ int count;
+
+ final NumericDocValues numericDocValues;
+ final SortedNumericDocValues sortedNumericDocValues;
+
+ private NumericDocValuesProducer() {
+ this.numericDocValues = new NumericDocValues() {
+ @Override
+ public long get(int docID) {
+ return dvLongValues[0];
+ }
+ };
+ this.sortedNumericDocValues = new SortedNumericDocValues() {
+ @Override
+ public void setDocument(int doc) {
+ }
+
+ @Override
+ public long valueAt(int index) {
+ return dvLongValues[index];
+ }
+
+ @Override
+ public int count() {
+ return count;
+ }
+ };
+ }
+
+ private void prepareForUsage() {
+ Arrays.sort(dvLongValues, 0, count);
+ }
+ }
+
/**
* Search support for Lucene framework integration; implements all methods
* required by the Lucene IndexReader contracts.
@@ -795,6 +980,9 @@ public class MemoryIndex {
private MemoryIndexReader() {
super(); // avoid as much superclass baggage as possible
+ for (Info info : fields.values()) {
+ info.prepareDocValues();
+ }
}
@Override
@@ -807,8 +995,18 @@ public class MemoryIndex {
removeCoreClosedListenerAsReaderClosedListener(this, listener);
}
- private Info getInfo(String fieldName) {
- return fields.get(fieldName);
+ private Info getInfoForExpectedDocValuesType(String fieldName, DocValuesType expectedType) {
+ if (expectedType == DocValuesType.NONE) {
+ return null;
+ }
+ Info info = fields.get(fieldName);
+ if (info == null) {
+ return null;
+ }
+ if (info.fieldInfo.getDocValuesType() != expectedType) {
+ return null;
+ }
+ return info;
}
@Override
@@ -828,32 +1026,87 @@ public class MemoryIndex {
@Override
public NumericDocValues getNumericDocValues(String field) {
- return null;
+ Info info = getInfoForExpectedDocValuesType(field, DocValuesType.NUMERIC);
+ if (info != null) {
+ return info.numericProducer.numericDocValues;
+ } else {
+ return null;
+ }
}
@Override
public BinaryDocValues getBinaryDocValues(String field) {
- return null;
+ return getSortedDocValues(field, DocValuesType.BINARY);
}
@Override
public SortedDocValues getSortedDocValues(String field) {
- return null;
+ return getSortedDocValues(field, DocValuesType.SORTED);
+ }
+
+ private SortedDocValues getSortedDocValues(String field, DocValuesType docValuesType) {
+ Info info = getInfoForExpectedDocValuesType(field, docValuesType);
+ if (info != null) {
+ return info.binaryProducer.sortedDocValues;
+ } else {
+ return null;
+ }
}
@Override
public SortedNumericDocValues getSortedNumericDocValues(String field) {
- return null;
+ Info info = getInfoForExpectedDocValuesType(field, DocValuesType.SORTED_NUMERIC);
+ if (info != null) {
+ return info.numericProducer.sortedNumericDocValues;
+ } else {
+ return null;
+ }
}
@Override
public SortedSetDocValues getSortedSetDocValues(String field) {
- return null;
+ Info info = getInfoForExpectedDocValuesType(field, DocValuesType.SORTED_SET);
+ if (info != null) {
+ return new SortedSetDocValues() {
+
+ int index = 0;
+
+ @Override
+ public long nextOrd() {
+ if (index >= info.binaryProducer.dvBytesValuesSet.size()) {
+ return NO_MORE_ORDS;
+ }
+ return index++;
+ }
+
+ @Override
+ public void setDocument(int docID) {
+ index = 0;
+ }
+
+ @Override
+ public BytesRef lookupOrd(long ord) {
+ return info.binaryProducer.getValue((int) ord);
+ }
+
+ @Override
+ public long getValueCount() {
+ return info.binaryProducer.dvBytesValuesSet.size();
+ }
+ };
+ } else {
+ return null;
+ }
}
@Override
public Bits getDocsWithField(String field) throws IOException {
- return null;
+ Info info = fields.get(field);
+ if (info != null && info.fieldInfo.getDocValuesType() != DocValuesType.NONE) {
+ return new Bits.MatchAllBits(1);
+ } else {
+ return null;
+ }
}
@Override
@@ -866,7 +1119,25 @@ public class MemoryIndex {
// no-op
}
+ @Override
+ public Fields fields() {
+ Map<String, Info> filteredFields = fields.entrySet().stream()
+ .filter(entry -> entry.getValue().numTokens > 0)
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,
+ (u,v) -> { throw new IllegalStateException(String.format(Locale.ROOT, "Duplicate key %s", u));},
+ TreeMap::new
+ ));
+ return new MemoryFields(filteredFields );
+ }
+
private class MemoryFields extends Fields {
+
+ private final Map<String, Info> fields;
+
+ public MemoryFields(Map<String, Info> fields) {
+ this.fields = fields;
+ }
+
@Override
public Iterator<String> iterator() {
return fields.keySet().iterator();
@@ -875,8 +1146,9 @@ public class MemoryIndex {
@Override
public Terms terms(final String field) {
final Info info = fields.get(field);
- if (info == null)
+ if (info == null) {
return null;
+ }
return new Terms() {
@Override
@@ -932,11 +1204,6 @@ public class MemoryIndex {
return fields.size();
}
}
-
- @Override
- public Fields fields() {
- return new MemoryFields();
- }
private class MemoryTermsEnum extends TermsEnum {
private final Info info;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf3eea26/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
index b150ea3..7282e0e 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
@@ -21,13 +21,25 @@ import java.io.IOException;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockPayloadAnalyzer;
+import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
@@ -35,6 +47,7 @@ import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.ClassicSimilarity;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.junit.Before;
@@ -176,5 +189,125 @@ public class TestMemoryIndex extends LuceneTestCase {
}
+ public void testDocValues() throws Exception {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("numeric", 29L));
+ doc.add(new SortedNumericDocValuesField("sorted_numeric", 33L));
+ doc.add(new SortedNumericDocValuesField("sorted_numeric", 32L));
+ doc.add(new SortedNumericDocValuesField("sorted_numeric", 32L));
+ doc.add(new SortedNumericDocValuesField("sorted_numeric", 31L));
+ doc.add(new SortedNumericDocValuesField("sorted_numeric", 30L));
+ doc.add(new BinaryDocValuesField("binary", new BytesRef("a")));
+ doc.add(new SortedDocValuesField("sorted", new BytesRef("b")));
+ doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("f")));
+ doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("d")));
+ doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("d")));
+ doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("c")));
+
+ MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
+ LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();
+ NumericDocValues numericDocValues = leafReader.getNumericDocValues("numeric");
+ assertEquals(29L, numericDocValues.get(0));
+ SortedNumericDocValues sortedNumericDocValues = leafReader.getSortedNumericDocValues("sorted_numeric");
+ sortedNumericDocValues.setDocument(0);
+ assertEquals(5, sortedNumericDocValues.count());
+ assertEquals(30L, sortedNumericDocValues.valueAt(0));
+ assertEquals(31L, sortedNumericDocValues.valueAt(1));
+ assertEquals(32L, sortedNumericDocValues.valueAt(2));
+ assertEquals(32L, sortedNumericDocValues.valueAt(3));
+ assertEquals(33L, sortedNumericDocValues.valueAt(4));
+ BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("binary");
+ assertEquals("a", binaryDocValues.get(0).utf8ToString());
+ SortedDocValues sortedDocValues = leafReader.getSortedDocValues("sorted");
+ assertEquals("b", sortedDocValues.get(0).utf8ToString());
+ assertEquals(0, sortedDocValues.getOrd(0));
+ assertEquals("b", sortedDocValues.lookupOrd(0).utf8ToString());
+ SortedSetDocValues sortedSetDocValues = leafReader.getSortedSetDocValues("sorted_set");
+ assertEquals(3, sortedSetDocValues.getValueCount());
+ sortedSetDocValues.setDocument(0);
+ assertEquals(0L, sortedSetDocValues.nextOrd());
+ assertEquals(1L, sortedSetDocValues.nextOrd());
+ assertEquals(2L, sortedSetDocValues.nextOrd());
+ assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSetDocValues.nextOrd());
+ assertEquals("c", sortedSetDocValues.lookupOrd(0L).utf8ToString());
+ assertEquals("d", sortedSetDocValues.lookupOrd(1L).utf8ToString());
+ assertEquals("f", sortedSetDocValues.lookupOrd(2L).utf8ToString());
+ }
+
+ public void testInvalidDocValuesUsage() throws Exception {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("field", 29L));
+ doc.add(new BinaryDocValuesField("field", new BytesRef("30")));
+ try {
+ MemoryIndex.fromDocument(doc, analyzer);
+ } catch (IllegalArgumentException e) {
+ assertEquals("Can't add [BINARY] doc values field [field], because [NUMERIC] doc values field already exists", e.getMessage());
+ }
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("field", 29L));
+ doc.add(new NumericDocValuesField("field", 30L));
+ try {
+ MemoryIndex.fromDocument(doc, analyzer);
+ } catch (IllegalArgumentException e) {
+ assertEquals("Only one value per field allowed for [NUMERIC] doc values field [field]", e.getMessage());
+ }
+
+ doc = new Document();
+ doc.add(new TextField("field", "a b", Field.Store.NO));
+ doc.add(new BinaryDocValuesField("field", new BytesRef("a")));
+ doc.add(new BinaryDocValuesField("field", new BytesRef("b")));
+ try {
+ MemoryIndex.fromDocument(doc, analyzer);
+ } catch (IllegalArgumentException e) {
+ assertEquals("Only one value per field allowed for [BINARY] doc values field [field]", e.getMessage());
+ }
+
+ doc = new Document();
+ doc.add(new SortedDocValuesField("field", new BytesRef("a")));
+ doc.add(new SortedDocValuesField("field", new BytesRef("b")));
+ doc.add(new TextField("field", "a b", Field.Store.NO));
+ try {
+ MemoryIndex.fromDocument(doc, analyzer);
+ } catch (IllegalArgumentException e) {
+ assertEquals("Only one value per field allowed for [SORTED] doc values field [field]", e.getMessage());
+ }
+ }
+
+ public void testDocValuesDoNotAffectBoostPositionsOrOffset() throws Exception {
+ Document doc = new Document();
+ doc.add(new BinaryDocValuesField("text", new BytesRef("quick brown fox")));
+ doc.add(new TextField("text", "quick brown fox", Field.Store.NO));
+ MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer, true, true);
+ LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();
+ TermsEnum tenum = leafReader.terms("text").iterator();
+
+ assertEquals("brown", tenum.next().utf8ToString());
+ PostingsEnum penum = tenum.postings(null, PostingsEnum.OFFSETS);
+ assertEquals(0, penum.nextDoc());
+ assertEquals(1, penum.freq());
+ assertEquals(1, penum.nextPosition());
+ assertEquals(6, penum.startOffset());
+ assertEquals(11, penum.endOffset());
+
+ assertEquals("fox", tenum.next().utf8ToString());
+ penum = tenum.postings(penum, PostingsEnum.OFFSETS);
+ assertEquals(0, penum.nextDoc());
+ assertEquals(1, penum.freq());
+ assertEquals(2, penum.nextPosition());
+ assertEquals(12, penum.startOffset());
+ assertEquals(15, penum.endOffset());
+
+ assertEquals("quick", tenum.next().utf8ToString());
+ penum = tenum.postings(penum, PostingsEnum.OFFSETS);
+ assertEquals(0, penum.nextDoc());
+ assertEquals(1, penum.freq());
+ assertEquals(0, penum.nextPosition());
+ assertEquals(0, penum.startOffset());
+ assertEquals(5, penum.endOffset());
+
+ BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("text");
+ assertEquals("quick brown fox", binaryDocValues.get(0).utf8ToString());
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf3eea26/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
index 57e25fe..3e6778a 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
@@ -35,10 +35,18 @@ import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.LegacyLongField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.CompositeReader;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.Fields;
@@ -52,6 +60,9 @@ import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@@ -434,6 +445,129 @@ public class TestMemoryIndexAgainstRAMDir extends BaseTokenStreamTestCase {
assertNull(reader.terms("not-in-index"));
}
+ public void testDocValuesMemoryIndexVsNormalIndex() throws Exception {
+ Document doc = new Document();
+ long randomLong = random().nextLong();
+ doc.add(new NumericDocValuesField("numeric", randomLong));
+ if (random().nextBoolean()) {
+ doc.add(new LegacyLongField("numeric", randomLong, Field.Store.NO));
+ }
+ int numValues = atLeast(5);
+ for (int i = 0; i < numValues; i++) {
+ randomLong = random().nextLong();
+ doc.add(new SortedNumericDocValuesField("sorted_numeric", randomLong));
+ if (random().nextBoolean()) {
+ // randomly duplicate field/value
+ doc.add(new SortedNumericDocValuesField("sorted_numeric", randomLong));
+ }
+ if (random().nextBoolean()) {
+ doc.add(new LegacyLongField("numeric", randomLong, Field.Store.NO));
+ }
+ }
+ BytesRef randomTerm = new BytesRef(randomTerm());
+ doc.add(new BinaryDocValuesField("binary", randomTerm));
+ if (random().nextBoolean()) {
+ doc.add(new StringField("binary", randomTerm, Field.Store.NO));
+ }
+ randomTerm = new BytesRef(randomTerm());
+ doc.add(new SortedDocValuesField("sorted", randomTerm));
+ if (random().nextBoolean()) {
+ doc.add(new StringField("sorted", randomTerm, Field.Store.NO));
+ }
+ numValues = atLeast(5);
+ for (int i = 0; i < numValues; i++) {
+ randomTerm = new BytesRef(randomTerm());
+ doc.add(new SortedSetDocValuesField("sorted_set", randomTerm));
+ if (random().nextBoolean()) {
+ // randomly duplicate field/value
+ doc.add(new SortedSetDocValuesField("sorted_set", randomTerm));
+ }
+ if (random().nextBoolean()) {
+ // randomily just add a normal string field
+ doc.add(new StringField("sorted_set", randomTerm, Field.Store.NO));
+ }
+ }
+
+ MockAnalyzer mockAnalyzer = new MockAnalyzer(random());
+ MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, mockAnalyzer);
+ IndexReader indexReader = memoryIndex.createSearcher().getIndexReader();
+ LeafReader leafReader = indexReader.leaves().get(0).reader();
+
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer));
+ writer.addDocument(doc);
+ writer.close();
+ IndexReader controlIndexReader = DirectoryReader.open(dir);
+ LeafReader controlLeafReader = controlIndexReader.leaves().get(0).reader();
+
+ NumericDocValues numericDocValues = leafReader.getNumericDocValues("numeric");
+ NumericDocValues controlNumericDocValues = controlLeafReader.getNumericDocValues("numeric");
+ assertEquals(controlNumericDocValues.get(0), numericDocValues.get(0));
+
+ SortedNumericDocValues sortedNumericDocValues = leafReader.getSortedNumericDocValues("sorted_numeric");
+ sortedNumericDocValues.setDocument(0);
+ SortedNumericDocValues controlSortedNumericDocValues = controlLeafReader.getSortedNumericDocValues("sorted_numeric");
+ controlSortedNumericDocValues.setDocument(0);
+ assertEquals(controlSortedNumericDocValues.count(), sortedNumericDocValues.count());
+ for (int i = 0; i < controlSortedNumericDocValues.count(); i++) {
+ assertEquals(controlSortedNumericDocValues.valueAt(i), sortedNumericDocValues.valueAt(i));
+ }
+
+ BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("binary");
+ BinaryDocValues controlBinaryDocValues = controlLeafReader.getBinaryDocValues("binary");
+ assertEquals(controlBinaryDocValues.get(0), binaryDocValues.get(0));
+
+ SortedDocValues sortedDocValues = leafReader.getSortedDocValues("sorted");
+ SortedDocValues controlSortedDocValues = controlLeafReader.getSortedDocValues("sorted");
+ assertEquals(controlSortedDocValues.getValueCount(), sortedDocValues.getValueCount());
+ assertEquals(controlSortedDocValues.get(0), sortedDocValues.get(0));
+ assertEquals(controlSortedDocValues.getOrd(0), sortedDocValues.getOrd(0));
+ assertEquals(controlSortedDocValues.lookupOrd(0), sortedDocValues.lookupOrd(0));
+
+ SortedSetDocValues sortedSetDocValues = leafReader.getSortedSetDocValues("sorted_set");
+ sortedSetDocValues.setDocument(0);
+ SortedSetDocValues controlSortedSetDocValues = controlLeafReader.getSortedSetDocValues("sorted_set");
+ controlSortedSetDocValues.setDocument(0);
+ assertEquals(controlSortedSetDocValues.getValueCount(), sortedSetDocValues.getValueCount());
+ for (long controlOrd = controlSortedSetDocValues.nextOrd(); controlOrd != SortedSetDocValues.NO_MORE_ORDS;
+ controlOrd = controlSortedSetDocValues.nextOrd()) {
+ assertEquals(controlOrd, sortedSetDocValues.nextOrd());
+ assertEquals(controlSortedSetDocValues.lookupOrd(controlOrd), sortedSetDocValues.lookupOrd(controlOrd));
+ }
+ assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSetDocValues.nextOrd());
+
+ indexReader.close();
+ controlIndexReader.close();
+ dir.close();
+ }
+
+ public void testNormsWithDocValues() throws Exception {
+ MemoryIndex mi = new MemoryIndex(true, true);
+ MockAnalyzer mockAnalyzer = new MockAnalyzer(random());
+
+ mi.addField(new BinaryDocValuesField("text", new BytesRef("quick brown fox")), mockAnalyzer, 5f);
+ mi.addField(new TextField("text", "quick brown fox", Field.Store.NO), mockAnalyzer, 5f);
+ LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();
+
+ Document doc = new Document();
+ doc.add(new BinaryDocValuesField("text", new BytesRef("quick brown fox")));
+ Field field = new TextField("text", "quick brown fox", Field.Store.NO);
+ field.setBoost(5f);
+ doc.add(field);
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer));
+ writer.addDocument(doc);
+ writer.close();
+
+ IndexReader controlIndexReader = DirectoryReader.open(dir);
+ LeafReader controlLeafReader = controlIndexReader.leaves().get(0).reader();
+
+ assertEquals(controlLeafReader.getNormValues("text").get(0), leafReader.getNormValues("text").get(0));
+
+ controlIndexReader.close();
+ dir.close();
+ }
+
public void testDuellMemIndex() throws IOException {
LineFileDocs lineFileDocs = new LineFileDocs(random());
int numDocs = atLeast(10);
[04/50] lucene-solr:jira/SOLR-445: LUCENE-7099: use two-phase
iteration in LatLonPoint.newDistanceQuery
Posted by ho...@apache.org.
LUCENE-7099: use two-phase iteration in LatLonPoint.newDistanceQuery
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/576a4059
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/576a4059
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/576a4059
Branch: refs/heads/jira/SOLR-445
Commit: 576a40596db5f65f53f2d9e9c494fc71e7e7bade
Parents: 0ff341f
Author: Robert Muir <rm...@apache.org>
Authored: Sat Mar 12 10:58:32 2016 -0500
Committer: Robert Muir <rm...@apache.org>
Committed: Sat Mar 12 10:59:43 2016 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../org/apache/lucene/document/LatLonPoint.java | 14 +++--
.../document/LatLonPointDistanceQuery.java | 66 +++++++++++++++++---
.../apache/lucene/document/TestLatLonPoint.java | 10 +++
4 files changed, 80 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/576a4059/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 82d22fe..cd9fac2 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -18,6 +18,9 @@ Optimizations
* LUCENE-7071: Reduce bytes copying in OfflineSorter, giving ~10%
speedup on merging 2D LatLonPoint values (Mike McCandless)
+* LUCENE-7099: LatLonPoint's newDistanceQuery supports two-phase
+ iteration. (Robert Muir)
+
Other
* LUCENE-7087: Let MemoryIndex#fromDocument(...) accept 'Iterable<? extends IndexableField>'
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/576a4059/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
index e8c2f17..ebf850c 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java
@@ -96,8 +96,10 @@ public class LatLonPoint extends Field {
}
private static final int BITS = 32;
- private static final double LONGITUDE_SCALE = (0x1L<<BITS)/360.0D;
- private static final double LATITUDE_SCALE = (0x1L<<BITS)/180.0D;
+ private static final double LONGITUDE_ENCODE = (0x1L<<BITS)/360.0D;
+ private static final double LONGITUDE_DECODE = 1/LONGITUDE_ENCODE;
+ private static final double LATITUDE_ENCODE = (0x1L<<BITS)/180.0D;
+ private static final double LATITUDE_DECODE = 1/LATITUDE_ENCODE;
@Override
public String toString() {
@@ -142,7 +144,7 @@ public class LatLonPoint extends Field {
if (latitude == 90.0D) {
latitude = Math.nextDown(latitude);
}
- return Math.toIntExact((long) (latitude * LATITUDE_SCALE));
+ return Math.toIntExact((long) (latitude * LATITUDE_ENCODE));
}
/**
@@ -159,7 +161,7 @@ public class LatLonPoint extends Field {
if (longitude == 180.0D) {
longitude = Math.nextDown(longitude);
}
- return Math.toIntExact((long) (longitude * LONGITUDE_SCALE));
+ return Math.toIntExact((long) (longitude * LONGITUDE_ENCODE));
}
/**
@@ -168,7 +170,7 @@ public class LatLonPoint extends Field {
* @return decoded latitude value.
*/
public static double decodeLatitude(int encoded) {
- double result = encoded / LATITUDE_SCALE;
+ double result = encoded * LATITUDE_DECODE;
assert GeoUtils.isValidLat(result);
return result;
}
@@ -189,7 +191,7 @@ public class LatLonPoint extends Field {
* @return decoded longitude value.
*/
public static double decodeLongitude(int encoded) {
- double result = encoded / LONGITUDE_SCALE;
+ double result = encoded * LONGITUDE_DECODE;
assert GeoUtils.isValidLon(result);
return result;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/576a4059/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
index 9374b17..9d23986 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
@@ -18,22 +18,30 @@ package org.apache.lucene.document;
import java.io.IOException;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PointValues;
+import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.search.Weight;
import org.apache.lucene.spatial.util.GeoDistanceUtils;
import org.apache.lucene.spatial.util.GeoRect;
import org.apache.lucene.spatial.util.GeoUtils;
+import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.DocIdSetBuilder;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.SparseFixedBitSet;
/**
* Distance query for {@link LatLonPoint}.
@@ -95,22 +103,32 @@ final class LatLonPointDistanceQuery extends Query {
}
LatLonPoint.checkCompatible(fieldInfo);
+ // approximation (postfiltering has not yet been applied)
DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc());
+ // subset of documents that need no postfiltering, this is purely an optimization
+ final BitSet preApproved;
+ // dumb heuristic: if the field is really sparse, use a sparse impl
+ if (values.getDocCount(field) * 100L < reader.maxDoc()) {
+ preApproved = new SparseFixedBitSet(reader.maxDoc());
+ } else {
+ preApproved = new FixedBitSet(reader.maxDoc());
+ }
values.intersect(field,
new IntersectVisitor() {
@Override
+ public void grow(int count) {
+ result.grow(count);
+ }
+
+ @Override
public void visit(int docID) {
result.add(docID);
+ preApproved.set(docID);
}
@Override
public void visit(int docID, byte[] packedValue) {
- assert packedValue.length == 8;
- double lat = LatLonPoint.decodeLatitude(packedValue, 0);
- double lon = LatLonPoint.decodeLongitude(packedValue, Integer.BYTES);
- if (GeoDistanceUtils.haversin(latitude, longitude, lat, lon) <= radiusMeters) {
- visit(docID);
- }
+ result.add(docID);
}
// algorithm: we create a bounding box (two bounding boxes if we cross the dateline).
@@ -142,7 +160,41 @@ final class LatLonPointDistanceQuery extends Query {
}
});
- return new ConstantScoreScorer(this, score(), result.build().iterator());
+ DocIdSet set = result.build();
+ final DocIdSetIterator disi = set.iterator();
+ if (disi == null) {
+ return null;
+ }
+
+ // return two-phase iterator using docvalues to postfilter candidates
+ SortedNumericDocValues docValues = DocValues.getSortedNumeric(reader, field);
+ TwoPhaseIterator iterator = new TwoPhaseIterator(disi) {
+ @Override
+ public boolean matches() throws IOException {
+ int docId = disi.docID();
+ if (preApproved.get(docId)) {
+ return true;
+ } else {
+ docValues.setDocument(docId);
+ int count = docValues.count();
+ for (int i = 0; i < count; i++) {
+ long encoded = docValues.valueAt(i);
+ double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32));
+ double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF));
+ if (GeoDistanceUtils.haversin(latitude, longitude, docLatitude, docLongitude) <= radiusMeters) {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+
+ @Override
+ public float matchCost() {
+ return 20; // TODO: make this fancier
+ }
+ };
+ return new ConstantScoreScorer(this, score(), iterator);
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/576a4059/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java
index 519478f..d180d58 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java
@@ -159,6 +159,16 @@ public class TestLatLonPoint extends LuceneTestCase {
assertEquals(-180.0, LatLonPoint.decodeLongitude(LatLonPoint.encodeLongitude(-180.0)), ENCODING_TOLERANCE);
}
+ public void testEncodeDecodeExtremeValues() throws Exception {
+ assertEquals(Integer.MIN_VALUE, LatLonPoint.encodeLatitude(-90.0));
+ assertEquals(0, LatLonPoint.encodeLatitude(0.0));
+ assertEquals(Integer.MAX_VALUE, LatLonPoint.encodeLatitude(90.0));
+
+ assertEquals(Integer.MIN_VALUE, LatLonPoint.encodeLongitude(-180.0));
+ assertEquals(0, LatLonPoint.encodeLatitude(0.0));
+ assertEquals(Integer.MAX_VALUE, LatLonPoint.encodeLongitude(180.0));
+ }
+
public void testEncodeDecodeIsStable() throws Exception {
int iters = atLeast(1000);
for(int iter=0;iter<iters;iter++) {
[28/50] lucene-solr:jira/SOLR-445: LUCENE-7108: this test was running
the wrong range query
Posted by ho...@apache.org.
LUCENE-7108: this test was running the wrong range query
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3ba7456f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3ba7456f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3ba7456f
Branch: refs/heads/jira/SOLR-445
Commit: 3ba7456f7062f263dbb859cbe5d59046450a5371
Parents: c1dfeb8
Author: Mike McCandless <mi...@apache.org>
Authored: Tue Mar 15 09:50:12 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Tue Mar 15 09:50:12 2016 -0400
----------------------------------------------------------------------
lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3ba7456f/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
index f15ba19..2f3a3a6 100644
--- a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
+++ b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java
@@ -133,7 +133,7 @@ public class Test2BPoints extends LuceneTestCase {
w.forceMerge(1);
DirectoryReader r = DirectoryReader.open(w);
IndexSearcher s = new IndexSearcher(r);
- assertEquals(numDocs, s.count(LongPoint.newRangeQuery("long", new long[] {Long.MIN_VALUE, Long.MAX_VALUE}, new long[] {Long.MIN_VALUE, Long.MAX_VALUE})));
+ assertEquals(numDocs, s.count(LongPoint.newRangeQuery("long", new long[] {Long.MIN_VALUE, Long.MIN_VALUE}, new long[] {Long.MAX_VALUE, Long.MAX_VALUE})));
assertTrue(r.leaves().get(0).reader().getPointValues().size("long") > Integer.MAX_VALUE);
r.close();
w.close();
[18/50] lucene-solr:jira/SOLR-445: LUCENE-7099: improve test to
exercise searchAfter
Posted by ho...@apache.org.
LUCENE-7099: improve test to exercise searchAfter
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/80fe00ba
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/80fe00ba
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/80fe00ba
Branch: refs/heads/jira/SOLR-445
Commit: 80fe00ba18ac1f834028eb0064115a00bafe1f20
Parents: 95f20c6
Author: Robert Muir <rm...@apache.org>
Authored: Mon Mar 14 12:19:19 2016 -0400
Committer: Robert Muir <rm...@apache.org>
Committed: Mon Mar 14 12:19:19 2016 -0400
----------------------------------------------------------------------
.../document/TestLatLonPointDistanceSort.java | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/80fe00ba/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
index a0ee83f..ea36ea6 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java
@@ -27,7 +27,6 @@ import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.spatial.util.GeoDistanceUtils;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -59,7 +58,7 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
iw.close();
Sort sort = new Sort(LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731));
- TopFieldDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort);
+ TopDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort);
FieldDoc d = (FieldDoc) td.scoreDocs[0];
assertEquals(462.61748421408186D, (Double)d.fields[0], 0.0D);
@@ -130,6 +129,11 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
if (id != other.id) return false;
return true;
}
+
+ @Override
+ public String toString() {
+ return "Result [id=" + id + ", distance=" + distance + "]";
+ }
}
private void doRandomTest(int numDocs, int numQueries) throws IOException {
@@ -182,6 +186,17 @@ public class TestLatLonPointDistanceSort extends LuceneTestCase {
Result actual = new Result((Integer) fieldDoc.fields[1], (Double) fieldDoc.fields[0]);
assertEquals(expected[resultNumber], actual);
}
+
+ // get page2 with searchAfter()
+ if (topN < reader.maxDoc()) {
+ int page2 = TestUtil.nextInt(random(), 1, reader.maxDoc() - topN);
+ TopDocs topDocs2 = searcher.searchAfter(topDocs.scoreDocs[topN - 1], new MatchAllDocsQuery(), page2, sort);
+ for (int resultNumber = 0; resultNumber < page2; resultNumber++) {
+ FieldDoc fieldDoc = (FieldDoc) topDocs2.scoreDocs[resultNumber];
+ Result actual = new Result((Integer) fieldDoc.fields[1], (Double) fieldDoc.fields[0]);
+ assertEquals(expected[topN + resultNumber], actual);
+ }
+ }
}
reader.close();
writer.close();
[06/50] lucene-solr:jira/SOLR-445: don't wrap searchers in this test
Posted by ho...@apache.org.
don't wrap searchers in this test
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b420ad4e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b420ad4e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b420ad4e
Branch: refs/heads/jira/SOLR-445
Commit: b420ad4e3c9a99e9bf4f5577a45ccce1e3904571
Parents: 576a405
Author: Mike McCandless <mi...@apache.org>
Authored: Sun Mar 13 05:35:02 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Sun Mar 13 05:35:02 2016 -0400
----------------------------------------------------------------------
.../apache/lucene/queries/payloads/TestPayloadSpans.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b420ad4e/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java
index 179b971..9e9228b 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java
@@ -125,7 +125,7 @@ public class TestPayloadSpans extends LuceneTestCase {
IndexReader reader = getOnlyLeafReader(writer.getReader());
writer.close();
- checkSpans(snq.createWeight(newSearcher(reader), false).getSpans(reader.leaves().get(0), SpanWeight.Postings.PAYLOADS), 1, new int[]{2});
+ checkSpans(snq.createWeight(newSearcher(reader, false), false).getSpans(reader.leaves().get(0), SpanWeight.Postings.PAYLOADS), 1, new int[]{2});
reader.close();
directory.close();
}
@@ -264,7 +264,7 @@ public class TestPayloadSpans extends LuceneTestCase {
writer.addDocument(doc);
IndexReader reader = writer.getReader();
- IndexSearcher is = newSearcher(getOnlyLeafReader(reader));
+ IndexSearcher is = newSearcher(getOnlyLeafReader(reader), false);
writer.close();
SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a"));
@@ -303,7 +303,7 @@ public class TestPayloadSpans extends LuceneTestCase {
doc.add(new TextField("content", new StringReader("a b a d k f a h i k a k")));
writer.addDocument(doc);
IndexReader reader = writer.getReader();
- IndexSearcher is = newSearcher(getOnlyLeafReader(reader));
+ IndexSearcher is = newSearcher(getOnlyLeafReader(reader), false);
writer.close();
SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a"));
@@ -342,7 +342,7 @@ public class TestPayloadSpans extends LuceneTestCase {
doc.add(new TextField("content", new StringReader("j k a l f k k p a t a k l k t a")));
writer.addDocument(doc);
IndexReader reader = writer.getReader();
- IndexSearcher is = newSearcher(getOnlyLeafReader(reader));
+ IndexSearcher is = newSearcher(getOnlyLeafReader(reader), false);
writer.close();
SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a"));
@@ -438,7 +438,7 @@ public class TestPayloadSpans extends LuceneTestCase {
closeIndexReader = writer.getReader();
writer.close();
- IndexSearcher searcher = newSearcher(closeIndexReader);
+ IndexSearcher searcher = newSearcher(closeIndexReader, false);
return searcher;
}
[35/50] lucene-solr:jira/SOLR-445: SOLR-8814: Support GeoJSON
response format
Posted by ho...@apache.org.
SOLR-8814: Support GeoJSON response format
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/36145d02
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/36145d02
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/36145d02
Branch: refs/heads/jira/SOLR-445
Commit: 36145d02ccc838f50538a8b9d6ff9c68f3ccce22
Parents: 24830b7
Author: Ryan McKinley <ry...@apache.org>
Authored: Wed Mar 16 09:49:46 2016 -0700
Committer: Ryan McKinley <ry...@apache.org>
Committed: Wed Mar 16 10:06:49 2016 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 12 +
.../src/java/org/apache/solr/core/SolrCore.java | 3 +-
.../solr/response/GeoJSONResponseWriter.java | 345 +++++++++++++++++++
.../solr/response/JSONResponseWriter.java | 2 +-
.../transform/GeoTransformerFactory.java | 224 ++++++++++++
.../response/transform/TransformerFactory.java | 1 +
.../response/transform/WriteableGeoJSON.java | 55 +++
.../solr/schema/AbstractSpatialFieldType.java | 7 +
.../solr/collection1/conf/schema-spatial.xml | 1 +
.../response/TestGeoJSONResponseWriter.java | 279 +++++++++++++++
10 files changed, 927 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/36145d02/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 691e87f..c48032e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -34,6 +34,18 @@ Detailed Change List
New Features
----------------------
+* SOLR-8814: Support GeoJSON response writer and general spatial formatting. Adding
+ &wt=geojson&geojson.field=<your geometry field>
+ Will return a FeatureCollection for each SolrDocumentList and a Feature with the
+ requested geometry for each SolrDocument. The requested geometry field needs
+ to either extend AbstractSpatialFieldType or store a GeoJSON string. This also adds
+ a [geo] DocumentTransformer that can return the Shape in a variety of formats:
+ &fl=[geo f=<your geometry field> w=(GeoJSON|WKT|POLY)]
+ The default format is GeoJSON. For information on the supported formats, see:
+ https://github.com/locationtech/spatial4j/blob/master/FORMATS.md
+ To return the FeatureCollection as the root element, add '&omitHeader=true" (ryan)
+
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/36145d02/solr/core/src/java/org/apache/solr/core/SolrCore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 7a65a72..c5e54d2 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -2123,10 +2123,11 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
private final PluginBag<QueryResponseWriter> responseWriters = new PluginBag<>(QueryResponseWriter.class, this);
public static final Map<String ,QueryResponseWriter> DEFAULT_RESPONSE_WRITERS ;
static{
- HashMap<String, QueryResponseWriter> m= new HashMap<>(14, 1);
+ HashMap<String, QueryResponseWriter> m= new HashMap<>(15, 1);
m.put("xml", new XMLResponseWriter());
m.put("standard", m.get("xml"));
m.put(CommonParams.JSON, new JSONResponseWriter());
+ m.put("geojson", new GeoJSONResponseWriter());
m.put("python", new PythonResponseWriter());
m.put("php", new PHPResponseWriter());
m.put("phps", new PHPSerializedResponseWriter());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/36145d02/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java b/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java
new file mode 100644
index 0000000..896be92
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.response;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.lucene.index.IndexableField;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.transform.GeoTransformerFactory;
+import org.apache.solr.response.transform.WriteableGeoJSON;
+import org.apache.solr.schema.AbstractSpatialFieldType;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.ReturnFields;
+import org.locationtech.spatial4j.context.SpatialContext;
+import org.locationtech.spatial4j.io.ShapeWriter;
+import org.locationtech.spatial4j.io.SupportedFormats;
+import org.locationtech.spatial4j.shape.Shape;
+
+/**
+ * Extend the standard JSONResponseWriter to support GeoJSON. This writes
+ * a {@link SolrDocumentList} with a 'FeatureCollection', following the
+ * specification in <a href="http://geojson.org/">geojson.org</a>
+ */
+public class GeoJSONResponseWriter extends JSONResponseWriter {
+
+ public static final String FIELD = "geojson.field";
+
+ @Override
+ public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
+
+ String geofield = req.getParams().get(FIELD, null);
+ if(geofield==null || geofield.length()==0) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "GeoJSON. Missing parameter: '"+FIELD+"'");
+ }
+
+ SchemaField sf = req.getSchema().getFieldOrNull(geofield);
+ if(sf==null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "GeoJSON. Unknown field: '"+FIELD+"'="+geofield);
+ }
+
+ SupportedFormats formats = null;
+ if(sf.getType() instanceof AbstractSpatialFieldType) {
+ SpatialContext ctx = ((AbstractSpatialFieldType)sf.getType()).getSpatialContext();
+ formats = ctx.getFormats();
+ }
+
+ JSONWriter w = new GeoJSONWriter(writer, req, rsp,
+ geofield,
+ formats);
+
+ try {
+ w.writeResponse();
+ } finally {
+ w.close();
+ }
+ }
+}
+
+class GeoJSONWriter extends JSONWriter {
+
+ final SupportedFormats formats;
+ final ShapeWriter geowriter;
+ final String geofield;
+
+ public GeoJSONWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp,
+ String geofield, SupportedFormats formats) {
+ super(writer, req, rsp);
+ this.geofield = geofield;
+ this.formats = formats;
+ if(formats==null) {
+ this.geowriter = null;
+ }
+ else {
+ this.geowriter = formats.getGeoJsonWriter();
+ }
+ }
+
+ @Override
+ public void writeResponse() throws IOException {
+ if(req.getParams().getBool(CommonParams.OMIT_HEADER, false)) {
+ if(wrapperFunction!=null) {
+ writer.write(wrapperFunction + "(");
+ }
+ rsp.removeResponseHeader();
+
+ NamedList<Object> vals = rsp.getValues();
+ Object response = vals.remove("response");
+ if(vals.size()==0) {
+ writeVal(null, response);
+ }
+ else {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "GeoJSON with "+CommonParams.OMIT_HEADER +
+ " can not return more than a result set");
+ }
+
+ if(wrapperFunction!=null) {
+ writer.write(')');
+ }
+ writer.write('\n'); // ending with a newline looks much better from the command line
+ }
+ else {
+ super.writeResponse();
+ }
+ }
+
+ @Override
+ public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx) throws IOException {
+ if( idx > 0 ) {
+ writeArraySeparator();
+ }
+
+ indent();
+ writeMapOpener(-1);
+ incLevel();
+
+ writeKey("type", false);
+ writeVal(null, "Feature");
+
+ Object val = doc.getFieldValue(geofield);
+ if(val != null) {
+ writeFeatureGeometry(val);
+ }
+
+ boolean first=true;
+ for (String fname : doc.getFieldNames()) {
+ if (fname.equals(geofield) || ((returnFields!= null && !returnFields.wantsField(fname)))) {
+ continue;
+ }
+ writeMapSeparator();
+ if (first) {
+ indent();
+ writeKey("properties", false);
+ writeMapOpener(-1);
+ incLevel();
+
+ first=false;
+ }
+
+ indent();
+ writeKey(fname, true);
+ val = doc.getFieldValue(fname);
+
+ // SolrDocument will now have multiValued fields represented as a Collection,
+ // even if only a single value is returned for this document.
+ if (val instanceof List) {
+ // shortcut this common case instead of going through writeVal again
+ writeArray(name,((Iterable)val).iterator());
+ } else {
+ writeVal(fname, val);
+ }
+ }
+
+ // GeoJSON does not really support nested FeatureCollections
+ if(doc.hasChildDocuments()) {
+ if(first == false) {
+ writeMapSeparator();
+ indent();
+ }
+ writeKey("_childDocuments_", true);
+ writeArrayOpener(doc.getChildDocumentCount());
+ List<SolrDocument> childDocs = doc.getChildDocuments();
+ for(int i=0; i<childDocs.size(); i++) {
+ writeSolrDocument(null, childDocs.get(i), null, i);
+ }
+ writeArrayCloser();
+ }
+
+ // check that we added any properties
+ if(!first) {
+ decLevel();
+ writeMapCloser();
+ }
+
+ decLevel();
+ writeMapCloser();
+ }
+
+ protected void writeFeatureGeometry(Object geo) throws IOException
+ {
+ // Support multi-valued geometries
+ if(geo instanceof Iterable) {
+ Iterator iter = ((Iterable)geo).iterator();
+ if(!iter.hasNext()) {
+ return; // empty list
+ }
+ else {
+ geo = iter.next();
+
+ // More than value
+ if(iter.hasNext()) {
+ writeMapSeparator();
+ indent();
+ writeKey("geometry", false);
+ incLevel();
+
+ // TODO: in the future, we can be smart and try to make this the appropriate MULTI* value
+ // if all the values are the same
+ // { "type": "GeometryCollection",
+ // "geometries": [
+ writeMapOpener(-1);
+ writeKey("type",false);
+ writeStr(null, "GeometryCollection", false);
+ writeMapSeparator();
+ writeKey("geometries", false);
+ writeArrayOpener(-1); // no trivial way to determine array size
+ incLevel();
+
+ // The first one
+ indent();
+ writeGeo(geo);
+ while(iter.hasNext()) {
+ // Each element in the array
+ writeArraySeparator();
+ indent();
+ writeGeo(iter.next());
+ }
+
+ decLevel();
+ writeArrayCloser();
+ writeMapCloser();
+
+ decLevel();
+ return;
+ }
+ }
+ }
+
+ // Single Value
+ if(geo!=null) {
+ writeMapSeparator();
+ indent();
+ writeKey("geometry", false);
+ writeGeo(geo);
+ }
+ }
+
+ protected void writeGeo(Object geo) throws IOException {
+ Shape shape = null;
+ String str = null;
+ if(geo instanceof Shape) {
+ shape = (Shape)geo;
+ }
+ else if(geo instanceof IndexableField) {
+ str = ((IndexableField)geo).stringValue();
+ }
+ else if(geo instanceof WriteableGeoJSON) {
+ shape = ((WriteableGeoJSON)geo).shape;
+ }
+ else {
+ str = geo.toString();
+ }
+
+ if(str !=null) {
+ // Assume it is well formed JSON
+ if(str.startsWith("{") && str.endsWith("}")) {
+ writer.write(str);
+ return;
+ }
+
+ if(formats==null) {
+ // The check is here and not in the constructor because we do not know if the
+ // *stored* values for the field look like JSON until we actually try to read them
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "GeoJSON unable to write field: '&"+ GeoJSONResponseWriter.FIELD +"="+geofield+"' ("+str+")");
+ }
+ shape = formats.read(str);
+ }
+
+ if(geowriter==null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "GeoJSON unable to write field: '&"+ GeoJSONResponseWriter.FIELD +"="+geofield+"'");
+ }
+
+ if(shape!=null) {
+ geowriter.write(writer, shape);
+ }
+ }
+
+ @Override
+ public void writeStartDocumentList(String name,
+ long start, int size, long numFound, Float maxScore) throws IOException
+ {
+ writeMapOpener((maxScore==null) ? 3 : 4);
+ incLevel();
+ writeKey("type",false);
+ writeStr(null, "FeatureCollection", false);
+ writeMapSeparator();
+ writeKey("numFound",false);
+ writeLong(null,numFound);
+ writeMapSeparator();
+ writeKey("start",false);
+ writeLong(null,start);
+
+ if (maxScore!=null) {
+ writeMapSeparator();
+ writeKey("maxScore",false);
+ writeFloat(null,maxScore);
+ }
+ writeMapSeparator();
+
+ // if can we get bbox of all results, we should write it here
+
+ // indent();
+ writeKey("features",false);
+ writeArrayOpener(size);
+
+ incLevel();
+ }
+
+ @Override
+ public void writeEndDocumentList() throws IOException
+ {
+ decLevel();
+ writeArrayCloser();
+
+ decLevel();
+ indent();
+ writeMapCloser();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/36145d02/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java b/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java
index cf894b8..d257f57 100644
--- a/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java
@@ -70,8 +70,8 @@ public class JSONResponseWriter implements QueryResponseWriter {
}
class JSONWriter extends TextResponseWriter {
+ protected String wrapperFunction;
private String namedListStyle;
- private String wrapperFunction;
private static final String JSON_NL_STYLE="json.nl";
private static final String JSON_NL_MAP="map";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/36145d02/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java
new file mode 100644
index 0000000..7b7974b
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.response.transform;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.spatial.SpatialStrategy;
+import org.apache.lucene.spatial.composite.CompositeSpatialStrategy;
+import org.apache.lucene.spatial.serialized.SerializedDVStrategy;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.JSONResponseWriter;
+import org.apache.solr.response.QueryResponseWriter;
+import org.apache.solr.schema.AbstractSpatialFieldType;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.SyntaxError;
+import org.locationtech.spatial4j.io.GeoJSONWriter;
+import org.locationtech.spatial4j.io.ShapeWriter;
+import org.locationtech.spatial4j.io.SupportedFormats;
+import org.locationtech.spatial4j.shape.Shape;
+
+
+/**
+ * This DocumentTransformer will write a {@link Shape} to the SolrDocument using
+ * the requested format. Supported formats include:
+ * <ul>
+ * <li>GeoJSON</li>
+ * <li>WKT</li>
+ * <li>Polyshape</li>
+ * </ul>
+ * For more information see: <a href="https://github.com/locationtech/spatial4j/blob/master/FORMATS.md">spatial4j/FORMATS.md</a>
+ *
+ * The shape is either read from a stored field, or a ValueSource.
+ *
+ * This transformer is useful when:
+ * <ul>
+ * <li>You want to return a format different than the stored encoding (WKT vs GeoJSON)</li>
+ * <li>The {@link Shape} is stored in a {@link ValueSource}, not a stored field</li>
+ * <li>the value is not stored in a format the output understands (ie, raw GeoJSON)</li>
+ * </ul>
+ *
+ */
+public class GeoTransformerFactory extends TransformerFactory
+{
+ @Override
+ public DocTransformer create(String display, SolrParams params, SolrQueryRequest req) {
+
+ String fname = params.get("f", display);
+ if(fname.startsWith("[") && fname.endsWith("]")) {
+ fname = display.substring(1,display.length()-1);
+ }
+ SchemaField sf = req.getSchema().getFieldOrNull(fname);
+ if(sf==null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ this.getClass().getSimpleName() +" using unknown field: "+fname);
+ }
+ if(!(sf.getType() instanceof AbstractSpatialFieldType)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "GeoTransformer requested non-spatial field: "+fname + " ("+sf.getType().getClass().getSimpleName()+")");
+ }
+
+ final GeoFieldUpdater updater = new GeoFieldUpdater();
+ updater.field = fname;
+ updater.display = display;
+ updater.display_error = display+"_error";
+
+ ValueSource shapes = null;
+ AbstractSpatialFieldType<?> sdv = (AbstractSpatialFieldType<?>)sf.getType();
+ SpatialStrategy strategy = sdv.getStrategy(fname);
+ if(strategy instanceof CompositeSpatialStrategy) {
+ shapes = ((CompositeSpatialStrategy)strategy)
+ .getGeometryStrategy().makeShapeValueSource();
+ }
+ else if(strategy instanceof SerializedDVStrategy) {
+ shapes = ((SerializedDVStrategy)strategy)
+ .makeShapeValueSource();
+ }
+
+
+ String writerName = params.get("w", "GeoJSON");
+ updater.formats = strategy.getSpatialContext().getFormats();
+ updater.writer = updater.formats.getWriter(writerName);
+ if(updater.writer==null) {
+ StringBuilder str = new StringBuilder();
+ str.append( "Unknown Spatial Writer: " ).append(writerName);
+ str.append(" [");
+ for(ShapeWriter w : updater.formats.getWriters()) {
+ str.append(w.getFormatName()).append(' ');
+ }
+ str.append("]");
+ throw new SolrException(ErrorCode.BAD_REQUEST, str.toString());
+ }
+
+ QueryResponseWriter qw = req.getCore().getQueryResponseWriter(req);
+ updater.isJSON =
+ (qw.getClass() == JSONResponseWriter.class) &&
+ (updater.writer instanceof GeoJSONWriter);
+
+
+ // Using ValueSource
+ if(shapes!=null) {
+ // we don't really need the qparser... just so we can reuse valueSource
+ QParser parser = new QParser(null,null,params, req) {
+ @Override
+ public Query parse() throws SyntaxError {
+ return new MatchAllDocsQuery();
+ }
+ };
+
+ return new ValueSourceAugmenter(display, parser, shapes) {
+ @Override
+ protected void setValue(SolrDocument doc, Object val) {
+ updater.setValue(doc, val);
+ }
+ };
+ }
+
+ // Using the raw stored values
+ return new DocTransformer() {
+
+ @Override
+ public void transform(SolrDocument doc, int docid, float score) throws IOException {
+ Object val = doc.remove(updater.field);
+ if(val!=null) {
+ updater.setValue(doc, val);
+ }
+ }
+
+ @Override
+ public String getName() {
+ return updater.display;
+ }
+
+ @Override
+ public String[] getExtraRequestFields() {
+ return new String[] {updater.field};
+ }
+ };
+ }
+}
+
+class GeoFieldUpdater {
+ String field;
+ String display;
+ String display_error;
+
+ boolean isJSON;
+ ShapeWriter writer;
+ SupportedFormats formats;
+
+ void addShape(SolrDocument doc, Shape shape) {
+ if(isJSON) {
+ doc.addField(display, new WriteableGeoJSON(shape, writer));
+ }
+ else {
+ doc.addField(display, writer.toString(shape));
+ }
+ }
+
+ void setValue(SolrDocument doc, Object val) {
+ doc.remove(display);
+ if(val != null) {
+ if(val instanceof Iterable) {
+ Iterator iter = ((Iterable)val).iterator();
+ while(iter.hasNext()) {
+ addValue(doc, iter.next());
+ }
+ }
+ else {
+ addValue(doc, val);
+ }
+ }
+ }
+
+ void addValue(SolrDocument doc, Object val) {
+ if(val == null) {
+ return;
+ }
+
+ if(val instanceof Shape) {
+ addShape(doc, (Shape)val);
+ }
+ // Don't explode on 'InvalidShpae'
+ else if( val instanceof Exception) {
+ doc.setField( display_error, ((Exception)val).toString() );
+ }
+ else {
+ // Use the stored value
+ if(val instanceof IndexableField) {
+ val = ((IndexableField)val).stringValue();
+ }
+ try {
+ addShape(doc, formats.read(val.toString()));
+ }
+ catch(Exception ex) {
+ doc.setField( display_error, ex.toString() );
+ }
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/36145d02/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java
index a600adf..6e7a3dd 100644
--- a/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java
+++ b/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java
@@ -49,5 +49,6 @@ public abstract class TransformerFactory implements NamedListInitializedPlugin
defaultFactories.put( "child", new ChildDocTransformerFactory() );
defaultFactories.put( "json", new RawValueTransformerFactory("json") );
defaultFactories.put( "xml", new RawValueTransformerFactory("xml") );
+ defaultFactories.put( "geo", new GeoTransformerFactory() );
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/36145d02/solr/core/src/java/org/apache/solr/response/transform/WriteableGeoJSON.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/transform/WriteableGeoJSON.java b/solr/core/src/java/org/apache/solr/response/transform/WriteableGeoJSON.java
new file mode 100644
index 0000000..40acebf
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/response/transform/WriteableGeoJSON.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.response.transform;
+
+import java.io.IOException;
+
+import org.apache.solr.common.util.JavaBinCodec;
+import org.apache.solr.response.TextResponseWriter;
+import org.apache.solr.response.WriteableValue;
+import org.locationtech.spatial4j.io.ShapeWriter;
+import org.locationtech.spatial4j.shape.Shape;
+
+/**
+ * This will let the writer add values to the response directly
+ */
+public class WriteableGeoJSON extends WriteableValue {
+
+ public final Shape shape;
+ public final ShapeWriter jsonWriter;
+
+ public WriteableGeoJSON(Shape shape, ShapeWriter jsonWriter) {
+ this.shape = shape;
+ this.jsonWriter = jsonWriter;
+ }
+
+ @Override
+ public Object resolve(Object o, JavaBinCodec codec) throws IOException {
+ codec.writeStr(jsonWriter.toString(shape));
+ return null; // this means we wrote it
+ }
+
+ @Override
+ public void write(String name, TextResponseWriter writer) throws IOException {
+ jsonWriter.write(writer.getWriter(), shape);
+ }
+
+ @Override
+ public String toString() {
+ return jsonWriter.toString(shape);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/36145d02/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
index 222f0b8..e5fd8c6 100644
--- a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
@@ -390,6 +390,13 @@ public abstract class AbstractSpatialFieldType<T extends SpatialStrategy> extend
}
}
+ /**
+ * @return The Spatial Context for this field type
+ */
+ public SpatialContext getSpatialContext() {
+ return ctx;
+ }
+
@Override
public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException {
writer.writeStr(name, f.stringValue(), true);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/36145d02/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml b/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml
index 2c1ca1f..15837f3 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml
@@ -70,6 +70,7 @@
<field name="bbox" type="bbox" />
<dynamicField name="bboxD_*" type="bbox" indexed="true" />
+ <dynamicField name="str_*" type="string" indexed="true" stored="true"/>
</fields>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/36145d02/solr/core/src/test/org/apache/solr/response/TestGeoJSONResponseWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/response/TestGeoJSONResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestGeoJSONResponseWriter.java
new file mode 100644
index 0000000..191136b
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/response/TestGeoJSONResponseWriter.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.response;
+
+import java.lang.invoke.MethodHandles;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrException;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.locationtech.spatial4j.context.SpatialContext;
+import org.locationtech.spatial4j.io.SupportedFormats;
+import org.locationtech.spatial4j.shape.Shape;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class TestGeoJSONResponseWriter extends SolrTestCaseJ4 {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ final ObjectMapper jsonmapper = new ObjectMapper();
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-basic.xml","schema-spatial.xml");
+ createIndex();
+ }
+
+ public static void createIndex() {
+
+
+// <field name="srpt_geohash" type="srpt_geohash" multiValued="true" />
+// <field name="" type="srpt_quad" multiValued="true" />
+// <field name="" type="srpt_packedquad" multiValued="true" />
+// <field name="" type="stqpt_geohash" multiValued="true" />
+
+ // multiple valued field
+ assertU(adoc("id","H.A", "srpt_geohash","POINT( 1 2 )"));
+ assertU(adoc("id","H.B", "srpt_geohash","POINT( 1 2 )",
+ "srpt_geohash","POINT( 3 4 )"));
+ assertU(adoc("id","H.C", "srpt_geohash","LINESTRING (30 10, 10 30, 40 40)"));
+
+ assertU(adoc("id","Q.A", "srpt_quad","POINT( 1 2 )"));
+ assertU(adoc("id","Q.B", "srpt_quad","POINT( 1 2 )",
+ "srpt_quad","POINT( 3 4 )"));
+ assertU(adoc("id","Q.C", "srpt_quad","LINESTRING (30 10, 10 30, 40 40)"));
+
+ assertU(adoc("id","P.A", "srpt_packedquad","POINT( 1 2 )"));
+ assertU(adoc("id","P.B", "srpt_packedquad","POINT( 1 2 )",
+ "srpt_packedquad","POINT( 3 4 )"));
+ assertU(adoc("id","P.C", "srpt_packedquad","LINESTRING (30 10, 10 30, 40 40)"));
+
+
+ // single valued field
+ assertU(adoc("id","R.A", "srptgeom","POINT( 1 2 )"));
+
+ // non-spatial field
+ assertU(adoc("id","S.X", "str_shape","POINT( 1 2 )"));
+ assertU(adoc("id","S.A", "str_shape","{\"type\":\"Point\",\"coordinates\":[1,2]}"));
+
+
+ assertU(commit());
+ }
+
+ protected Map<String,Object> readJSON(String json) {
+ try {
+ return jsonmapper.readValue(json, Map.class);
+ }
+ catch(Exception ex) {
+ log.warn("Unable to read GeoJSON From: {}", json);
+ log.warn("Error", ex);
+ fail("Unable to parse JSON GeoJSON Response");
+ }
+ return null;
+ }
+
+ protected Map<String,Object> getFirstFeatureGeometry(Map<String,Object> json)
+ {
+ Map<String,Object> rsp = (Map<String,Object>)json.get("response");
+ assertEquals("FeatureCollection", rsp.get("type"));
+ List<Object> vals = (List<Object>)rsp.get("features");
+ assertEquals(1, vals.size());
+ Map<String,Object> feature = (Map<String,Object>)vals.get(0);
+ assertEquals("Feature", feature.get("type"));
+ return (Map<String,Object>)feature.get("geometry");
+ }
+
+ @Test
+ public void testRequestExceptions() throws Exception {
+
+ // Make sure we select the field
+ try {
+ h.query(req(
+ "q","*:*",
+ "wt","geojson",
+ "fl","*"));
+ fail("should Require a parameter to select the field");
+ }
+ catch(SolrException ex) {}
+
+
+ // non-spatial fields *must* be stored as JSON
+ try {
+ h.query(req(
+ "q","id:S.X",
+ "wt","geojson",
+ "fl","*",
+ "geojson.field", "str_shape"));
+ fail("should complain about bad shape config");
+ }
+ catch(SolrException ex) {}
+
+ }
+
+ @Test
+ public void testGeoJSONAtRoot() throws Exception {
+
+ // Try reading the whole resposne
+ String json = h.query(req(
+ "q","*:*",
+ "wt","geojson",
+ "rows","2",
+ "fl","*",
+ "geojson.field", "stqpt_geohash",
+ "indent","true"));
+
+ // Check that we have a normal solr response with 'responseHeader' and 'response'
+ Map<String,Object> rsp = readJSON(json);
+ assertNotNull(rsp.get("responseHeader"));
+ assertNotNull(rsp.get("response"));
+
+ json = h.query(req(
+ "q","*:*",
+ "wt","geojson",
+ "rows","2",
+ "fl","*",
+ "omitHeader", "true",
+ "geojson.field", "stqpt_geohash",
+ "indent","true"));
+
+ // Check that we have a normal solr response with 'responseHeader' and 'response'
+ rsp = readJSON(json);
+ assertNull(rsp.get("responseHeader"));
+ assertNull(rsp.get("response"));
+ assertEquals("FeatureCollection", rsp.get("type"));
+ assertNotNull(rsp.get("features"));
+ }
+
+ @Test
+ public void testGeoJSONOutput() throws Exception {
+
+ // Try reading the whole resposne
+ readJSON(h.query(req(
+ "q","*:*",
+ "wt","geojson",
+ "fl","*",
+ "geojson.field", "stqpt_geohash",
+ "indent","true")));
+
+ // Multivalued Valued Point
+ Map<String,Object> json = readJSON(h.query(req(
+ "q","id:H.B",
+ "wt","geojson",
+ "fl","*",
+ "geojson.field", "srpt_geohash",
+ "indent","true")));
+
+ Map<String,Object> geo = getFirstFeatureGeometry(json);
+ assertEquals( // NOTE: not actual JSON, it is Map.toString()!
+ "{type=GeometryCollection, geometries=["
+ + "{type=Point, coordinates=[1, 2]}, "
+ + "{type=Point, coordinates=[3, 4]}]}", ""+geo);
+
+
+ // Check the same value encoded on different field types
+ String[][] check = new String[][] {
+ { "id:H.A", "srpt_geohash" },
+ { "id:Q.A", "srpt_quad" },
+ { "id:P.A", "srpt_packedquad" },
+ { "id:R.A", "srptgeom" },
+ { "id:S.A", "str_shape" },
+ };
+
+ for(String[] args : check) {
+ json = readJSON(h.query(req(
+ "q",args[0],
+ "wt","geojson",
+ "fl","*",
+ "geojson.field", args[1])));
+
+ geo = getFirstFeatureGeometry(json);
+ assertEquals(
+ "Error reading point from: "+args[1] + " ("+args[0]+")",
+ // NOTE: not actual JSON, it is Map.toString()!
+ "{type=Point, coordinates=[1, 2]}", ""+geo);
+ }
+ }
+
+ protected Map<String,Object> readFirstDoc(String json)
+ {
+ List docs = (List)((Map)readJSON(json).get("response")).get("docs");
+ return (Map)docs.get(0);
+ }
+
+ public static String normalizeMapToJSON(String val) {
+ val = val.replace("\"", ""); // remove quotes
+ val = val.replace(':', '=');
+ val = val.replace(", ", ",");
+ return val;
+ }
+
+ @Test
+ public void testTransformToAllFormats() throws Exception {
+
+ String wkt = "POINT( 1 2 )";
+ SupportedFormats fmts = SpatialContext.GEO.getFormats();
+ Shape shape = fmts.read(wkt);
+
+ String[] check = new String[] {
+ "srpt_geohash",
+ "srpt_geohash",
+ "srpt_quad",
+ "srpt_packedquad",
+ "srptgeom",
+ // "str_shape", // NEEDS TO BE A SpatialField!
+ };
+
+ String[] checkFormats = new String[] {
+ "GeoJSON",
+ "WKT",
+ "POLY"
+ };
+
+ for(String field : check) {
+ // Add a document with the given field
+ assertU(adoc("id","test",
+ field, wkt));
+ assertU(commit());
+
+
+ for(String fmt : checkFormats) {
+ String json = h.query(req(
+ "q","id:test",
+ "wt","json",
+ "indent", "true",
+ "fl","xxx:[geo f="+field+" w="+fmt+"]"
+ ));
+
+ Map<String,Object> doc = readFirstDoc(json);
+ Object v = doc.get("xxx");
+ String expect = fmts.getWriter(fmt).toString(shape);
+
+ if(!(v instanceof String)) {
+ v = normalizeMapToJSON(v.toString());
+ expect = normalizeMapToJSON(expect);
+ }
+
+ assertEquals("Bad result: "+field+"/"+fmt, expect, v.toString());
+ }
+ }
+ }
+}