You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/03/06 14:26:54 UTC
[2/4] lucene-solr git commit: LUCENE-7070: insert missing 's' in
PointFormat/Reader/Writer
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74e7e150/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
index 9aafb3f..9faa0bc 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
@@ -22,11 +22,11 @@ import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.PointFormat;
-import org.apache.lucene.codecs.PointReader;
-import org.apache.lucene.codecs.PointWriter;
-import org.apache.lucene.codecs.lucene60.Lucene60PointReader;
-import org.apache.lucene.codecs.lucene60.Lucene60PointWriter;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PointsReader;
+import org.apache.lucene.codecs.PointsWriter;
+import org.apache.lucene.codecs.lucene60.Lucene60PointsReader;
+import org.apache.lucene.codecs.lucene60.Lucene60PointsWriter;
import org.apache.lucene.document.BinaryPoint;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoublePoint;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74e7e150/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
index 66a1593..cd0d719 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
@@ -32,11 +32,11 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.PointFormat;
-import org.apache.lucene.codecs.PointReader;
-import org.apache.lucene.codecs.PointWriter;
-import org.apache.lucene.codecs.lucene60.Lucene60PointReader;
-import org.apache.lucene.codecs.lucene60.Lucene60PointWriter;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PointsReader;
+import org.apache.lucene.codecs.PointsWriter;
+import org.apache.lucene.codecs.lucene60.Lucene60PointsReader;
+import org.apache.lucene.codecs.lucene60.Lucene60PointsWriter;
import org.apache.lucene.document.BinaryPoint;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoublePoint;
@@ -1157,21 +1157,21 @@ public class TestPointQueries extends LuceneTestCase {
int maxPointsInLeafNode = TestUtil.nextInt(random(), 16, 2048);
double maxMBSortInHeap = 4.0 + (3*random().nextDouble());
if (VERBOSE) {
- System.out.println("TEST: using Lucene60PointFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap);
+ System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap);
}
return new FilterCodec("Lucene60", Codec.getDefault()) {
@Override
- public PointFormat pointFormat() {
- return new PointFormat() {
+ public PointsFormat pointsFormat() {
+ return new PointsFormat() {
@Override
- public PointWriter fieldsWriter(SegmentWriteState writeState) throws IOException {
- return new Lucene60PointWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap);
+ public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException {
+ return new Lucene60PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap);
}
@Override
- public PointReader fieldsReader(SegmentReadState readState) throws IOException {
- return new Lucene60PointReader(readState);
+ public PointsReader fieldsReader(SegmentReadState readState) throws IOException {
+ return new Lucene60PointsReader(readState);
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74e7e150/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java
index e37d75e..fa95710 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java
@@ -20,11 +20,11 @@ import java.io.IOException;
import java.util.BitSet;
import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.PointFormat;
-import org.apache.lucene.codecs.PointReader;
-import org.apache.lucene.codecs.PointWriter;
-import org.apache.lucene.codecs.lucene60.Lucene60PointReader;
-import org.apache.lucene.codecs.lucene60.Lucene60PointWriter;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PointsReader;
+import org.apache.lucene.codecs.PointsWriter;
+import org.apache.lucene.codecs.lucene60.Lucene60PointsReader;
+import org.apache.lucene.codecs.lucene60.Lucene60PointsWriter;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
@@ -36,8 +36,8 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.spatial.util.GeoDistanceUtils;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.bkd.BKDWriter;
/** Simple tests for {@link LatLonPoint#newDistanceQuery} */
@@ -119,16 +119,16 @@ public class TestLatLonPointDistanceQuery extends LuceneTestCase {
int pointsInLeaf = 2 + random().nextInt(4);
iwc.setCodec(new FilterCodec("Lucene60", TestUtil.getDefaultCodec()) {
@Override
- public PointFormat pointFormat() {
- return new PointFormat() {
+ public PointsFormat pointsFormat() {
+ return new PointsFormat() {
@Override
- public PointWriter fieldsWriter(SegmentWriteState writeState) throws IOException {
- return new Lucene60PointWriter(writeState, pointsInLeaf, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP);
+ public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException {
+ return new Lucene60PointsWriter(writeState, pointsInLeaf, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP);
}
@Override
- public PointReader fieldsReader(SegmentReadState readState) throws IOException {
- return new Lucene60PointReader(readState);
+ public PointsReader fieldsReader(SegmentReadState readState) throws IOException {
+ return new Lucene60PointsReader(readState);
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74e7e150/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java
index 675d642..9d00d3e 100644
--- a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java
@@ -27,12 +27,12 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.PointFormat;
-import org.apache.lucene.codecs.PointReader;
-import org.apache.lucene.codecs.PointWriter;
import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.lucene60.Lucene60PointReader;
-import org.apache.lucene.codecs.lucene60.Lucene60PointWriter;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PointsReader;
+import org.apache.lucene.codecs.PointsWriter;
+import org.apache.lucene.codecs.lucene60.Lucene60PointsReader;
+import org.apache.lucene.codecs.lucene60.Lucene60PointsWriter;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
@@ -76,21 +76,21 @@ public class TestGeo3DPoint extends LuceneTestCase {
int maxPointsInLeafNode = TestUtil.nextInt(random(), 16, 2048);
double maxMBSortInHeap = 3.0 + (3*random().nextDouble());
if (VERBOSE) {
- System.out.println("TEST: using Lucene60PointFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap);
+ System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap);
}
return new FilterCodec("Lucene60", Codec.getDefault()) {
@Override
- public PointFormat pointFormat() {
- return new PointFormat() {
+ public PointsFormat pointsFormat() {
+ return new PointsFormat() {
@Override
- public PointWriter fieldsWriter(SegmentWriteState writeState) throws IOException {
- return new Lucene60PointWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap);
+ public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException {
+ return new Lucene60PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap);
}
@Override
- public PointReader fieldsReader(SegmentReadState readState) throws IOException {
- return new Lucene60PointReader(readState);
+ public PointsReader fieldsReader(SegmentReadState readState) throws IOException {
+ return new Lucene60PointsReader(readState);
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74e7e150/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
index 9e06307..15bcfa2 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
@@ -16,11 +16,11 @@
*/
package org.apache.lucene.codecs.asserting;
-import org.apache.lucene.codecs.PointFormat;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.PointsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
@@ -53,7 +53,7 @@ public class AssertingCodec extends FilterCodec {
private final LiveDocsFormat liveDocs = new AssertingLiveDocsFormat();
private final PostingsFormat defaultFormat = new AssertingPostingsFormat();
private final DocValuesFormat defaultDVFormat = new AssertingDocValuesFormat();
- private final PointFormat pointFormat = new AssertingPointFormat();
+ private final PointsFormat pointsFormat = new AssertingPointsFormat();
public AssertingCodec() {
super("Asserting", TestUtil.getDefaultCodec());
@@ -90,8 +90,8 @@ public class AssertingCodec extends FilterCodec {
}
@Override
- public PointFormat pointFormat() {
- return pointFormat;
+ public PointsFormat pointsFormat() {
+ return pointsFormat;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74e7e150/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java
deleted file mode 100644
index 07365e0..0000000
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.codecs.asserting;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-
-import org.apache.lucene.codecs.PointFormat;
-import org.apache.lucene.codecs.PointReader;
-import org.apache.lucene.codecs.PointWriter;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.MergeState;
-import org.apache.lucene.index.PointValues.IntersectVisitor;
-import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.StringHelper;
-import org.apache.lucene.util.TestUtil;
-
-/**
- * Just like the default point format but with additional asserts.
- */
-
-public final class AssertingPointFormat extends PointFormat {
- private final PointFormat in;
-
- /** Create a new AssertingPointFormat */
- public AssertingPointFormat() {
- this(TestUtil.getDefaultCodec().pointFormat());
- }
-
- /**
- * Expert: Create an AssertingPointFormat.
- * This is only intended to pass special parameters for testing.
- */
- // TODO: can we randomize this a cleaner way? e.g. stored fields and vectors do
- // this with a separate codec...
- public AssertingPointFormat(PointFormat in) {
- this.in = in;
- }
-
- @Override
- public PointWriter fieldsWriter(SegmentWriteState state) throws IOException {
- return new AssertingPointWriter(state, in.fieldsWriter(state));
- }
-
- @Override
- public PointReader fieldsReader(SegmentReadState state) throws IOException {
- return new AssertingPointReader(state.segmentInfo.maxDoc(), in.fieldsReader(state));
- }
-
- /** Validates in the 1D case that all points are visited in order, and point values are in bounds of the last cell checked */
- static class AssertingIntersectVisitor implements IntersectVisitor {
- final IntersectVisitor in;
- final int numDims;
- final int bytesPerDim;
- final byte[] lastDocValue;
- final byte[] lastMinPackedValue;
- final byte[] lastMaxPackedValue;
- private Relation lastCompareResult;
- private int lastDocID = -1;
-
- public AssertingIntersectVisitor(int numDims, int bytesPerDim, IntersectVisitor in) {
- this.in = in;
- this.numDims = numDims;
- this.bytesPerDim = bytesPerDim;
- lastMaxPackedValue = new byte[numDims*bytesPerDim];
- lastMinPackedValue = new byte[numDims*bytesPerDim];
- if (numDims == 1) {
- lastDocValue = new byte[bytesPerDim];
- } else {
- lastDocValue = null;
- }
- }
-
- @Override
- public void visit(int docID) throws IOException {
- // This method, not filtering each hit, should only be invoked when the cell is inside the query shape:
- assert lastCompareResult == Relation.CELL_INSIDE_QUERY;
- in.visit(docID);
- }
-
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {
-
- // This method, to filter each doc's value, should only be invoked when the cell crosses the query shape:
- assert lastCompareResult == PointValues.Relation.CELL_CROSSES_QUERY;
-
- // This doc's packed value should be contained in the last cell passed to compare:
- for(int dim=0;dim<numDims;dim++) {
- assert StringHelper.compare(bytesPerDim, lastMinPackedValue, dim*bytesPerDim, packedValue, dim*bytesPerDim) <= 0: "dim=" + dim + " of " + numDims + " value=" + new BytesRef(packedValue);
- assert StringHelper.compare(bytesPerDim, lastMaxPackedValue, dim*bytesPerDim, packedValue, dim*bytesPerDim) >= 0: "dim=" + dim + " of " + numDims + " value=" + new BytesRef(packedValue);
- }
-
- // TODO: we should assert that this "matches" whatever relation the last call to compare had returned
- assert packedValue.length == numDims * bytesPerDim;
- if (numDims == 1) {
- int cmp = StringHelper.compare(bytesPerDim, lastDocValue, 0, packedValue, 0);
- if (cmp < 0) {
- // ok
- } else if (cmp == 0) {
- assert lastDocID <= docID: "doc ids are out of order when point values are the same!";
- } else {
- // out of order!
- assert false: "point values are out of order";
- }
- System.arraycopy(packedValue, 0, lastDocValue, 0, bytesPerDim);
- }
- in.visit(docID, packedValue);
- }
-
- @Override
- public void grow(int count) {
- in.grow(count);
- }
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- for(int dim=0;dim<numDims;dim++) {
- assert StringHelper.compare(bytesPerDim, minPackedValue, dim*bytesPerDim, maxPackedValue, dim*bytesPerDim) <= 0;
- }
- System.arraycopy(maxPackedValue, 0, lastMaxPackedValue, 0, numDims*bytesPerDim);
- System.arraycopy(minPackedValue, 0, lastMinPackedValue, 0, numDims*bytesPerDim);
- lastCompareResult = in.compare(minPackedValue, maxPackedValue);
- return lastCompareResult;
- }
- }
-
- static class AssertingPointReader extends PointReader {
- private final PointReader in;
- private final int maxDoc;
-
- AssertingPointReader(int maxDoc, PointReader in) {
- this.in = in;
- this.maxDoc = maxDoc;
- // do a few simple checks on init
- assert toString() != null;
- assert ramBytesUsed() >= 0;
- assert getChildResources() != null;
- }
-
- @Override
- public void close() throws IOException {
- in.close();
- in.close(); // close again
- }
-
- @Override
- public void intersect(String fieldName, IntersectVisitor visitor) throws IOException {
- in.intersect(fieldName,
- new AssertingIntersectVisitor(in.getNumDimensions(fieldName), in.getBytesPerDimension(fieldName), visitor));
- }
-
- @Override
- public long ramBytesUsed() {
- long v = in.ramBytesUsed();
- assert v >= 0;
- return v;
- }
-
- @Override
- public Collection<Accountable> getChildResources() {
- Collection<Accountable> res = in.getChildResources();
- TestUtil.checkReadOnly(res);
- return res;
- }
-
- @Override
- public void checkIntegrity() throws IOException {
- in.checkIntegrity();
- }
-
- @Override
- public PointReader getMergeInstance() throws IOException {
- return new AssertingPointReader(maxDoc, in.getMergeInstance());
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName() + "(" + in.toString() + ")";
- }
-
- @Override
- public byte[] getMinPackedValue(String fieldName) throws IOException {
- assertStats(fieldName);
- return in.getMinPackedValue(fieldName);
- }
-
- @Override
- public byte[] getMaxPackedValue(String fieldName) throws IOException {
- assertStats(fieldName);
- return in.getMaxPackedValue(fieldName);
- }
-
- @Override
- public int getNumDimensions(String fieldName) throws IOException {
- assertStats(fieldName);
- return in.getNumDimensions(fieldName);
- }
-
- @Override
- public int getBytesPerDimension(String fieldName) throws IOException {
- assertStats(fieldName);
- return in.getBytesPerDimension(fieldName);
- }
-
- @Override
- public long size(String fieldName) {
- assertStats(fieldName);
- return in.size(fieldName);
- }
-
- @Override
- public int getDocCount(String fieldName) {
- assertStats(fieldName);
- return in.getDocCount(fieldName);
- }
-
- private void assertStats(String fieldName) {
- assert in.size(fieldName) >= 0;
- assert in.getDocCount(fieldName) >= 0;
- assert in.getDocCount(fieldName) <= in.size(fieldName);
- assert in.getDocCount(fieldName) <= maxDoc;
- }
- }
-
- static class AssertingPointWriter extends PointWriter {
- private final PointWriter in;
-
- AssertingPointWriter(SegmentWriteState writeState, PointWriter in) {
- this.in = in;
- }
-
- @Override
- public void writeField(FieldInfo fieldInfo, PointReader values) throws IOException {
- if (fieldInfo.getPointDimensionCount() == 0) {
- throw new IllegalArgumentException("writing field=\"" + fieldInfo.name + "\" but pointDimensionalCount is 0");
- }
- in.writeField(fieldInfo, values);
- }
-
- @Override
- public void merge(MergeState mergeState) throws IOException {
- in.merge(mergeState);
- }
-
- @Override
- public void finish() throws IOException {
- in.finish();
- }
-
- @Override
- public void close() throws IOException {
- in.close();
- in.close(); // close again
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74e7e150/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
new file mode 100644
index 0000000..061d5b6
--- /dev/null
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.asserting;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PointsReader;
+import org.apache.lucene.codecs.PointsWriter;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.MergeState;
+import org.apache.lucene.index.PointValues.IntersectVisitor;
+import org.apache.lucene.index.PointValues.Relation;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.TestUtil;
+
+/**
+ * Just like the default point format but with additional asserts.
+ */
+
+public final class AssertingPointsFormat extends PointsFormat {
+ private final PointsFormat in;
+
+ /** Create a new AssertingPointsFormat */
+ public AssertingPointsFormat() {
+ this(TestUtil.getDefaultCodec().pointsFormat());
+ }
+
+ /**
+ * Expert: Create an AssertingPointsFormat.
+ * This is only intended to pass special parameters for testing.
+ */
+ // TODO: can we randomize this a cleaner way? e.g. stored fields and vectors do
+ // this with a separate codec...
+ public AssertingPointsFormat(PointsFormat in) {
+ this.in = in;
+ }
+
+ @Override
+ public PointsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+ return new AssertingPointsWriter(state, in.fieldsWriter(state));
+ }
+
+ @Override
+ public PointsReader fieldsReader(SegmentReadState state) throws IOException {
+ return new AssertingPointsReader(state.segmentInfo.maxDoc(), in.fieldsReader(state));
+ }
+
+ /** Validates in the 1D case that all points are visited in order, and point values are in bounds of the last cell checked */
+ static class AssertingIntersectVisitor implements IntersectVisitor {
+ final IntersectVisitor in;
+ final int numDims;
+ final int bytesPerDim;
+ final byte[] lastDocValue;
+ final byte[] lastMinPackedValue;
+ final byte[] lastMaxPackedValue;
+ private Relation lastCompareResult;
+ private int lastDocID = -1;
+
+ public AssertingIntersectVisitor(int numDims, int bytesPerDim, IntersectVisitor in) {
+ this.in = in;
+ this.numDims = numDims;
+ this.bytesPerDim = bytesPerDim;
+ lastMaxPackedValue = new byte[numDims*bytesPerDim];
+ lastMinPackedValue = new byte[numDims*bytesPerDim];
+ if (numDims == 1) {
+ lastDocValue = new byte[bytesPerDim];
+ } else {
+ lastDocValue = null;
+ }
+ }
+
+ @Override
+ public void visit(int docID) throws IOException {
+ // This method, not filtering each hit, should only be invoked when the cell is inside the query shape:
+ assert lastCompareResult == Relation.CELL_INSIDE_QUERY;
+ in.visit(docID);
+ }
+
+ @Override
+ public void visit(int docID, byte[] packedValue) throws IOException {
+
+ // This method, to filter each doc's value, should only be invoked when the cell crosses the query shape:
+ assert lastCompareResult == PointValues.Relation.CELL_CROSSES_QUERY;
+
+ // This doc's packed value should be contained in the last cell passed to compare:
+ for(int dim=0;dim<numDims;dim++) {
+ assert StringHelper.compare(bytesPerDim, lastMinPackedValue, dim*bytesPerDim, packedValue, dim*bytesPerDim) <= 0: "dim=" + dim + " of " + numDims + " value=" + new BytesRef(packedValue);
+ assert StringHelper.compare(bytesPerDim, lastMaxPackedValue, dim*bytesPerDim, packedValue, dim*bytesPerDim) >= 0: "dim=" + dim + " of " + numDims + " value=" + new BytesRef(packedValue);
+ }
+
+ // TODO: we should assert that this "matches" whatever relation the last call to compare had returned
+ assert packedValue.length == numDims * bytesPerDim;
+ if (numDims == 1) {
+ int cmp = StringHelper.compare(bytesPerDim, lastDocValue, 0, packedValue, 0);
+ if (cmp < 0) {
+ // ok
+ } else if (cmp == 0) {
+ assert lastDocID <= docID: "doc ids are out of order when point values are the same!";
+ } else {
+ // out of order!
+ assert false: "point values are out of order";
+ }
+ System.arraycopy(packedValue, 0, lastDocValue, 0, bytesPerDim);
+ }
+ in.visit(docID, packedValue);
+ }
+
+ @Override
+ public void grow(int count) {
+ in.grow(count);
+ }
+
+ @Override
+ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
+ for(int dim=0;dim<numDims;dim++) {
+ assert StringHelper.compare(bytesPerDim, minPackedValue, dim*bytesPerDim, maxPackedValue, dim*bytesPerDim) <= 0;
+ }
+ System.arraycopy(maxPackedValue, 0, lastMaxPackedValue, 0, numDims*bytesPerDim);
+ System.arraycopy(minPackedValue, 0, lastMinPackedValue, 0, numDims*bytesPerDim);
+ lastCompareResult = in.compare(minPackedValue, maxPackedValue);
+ return lastCompareResult;
+ }
+ }
+
+ static class AssertingPointsReader extends PointsReader {
+ private final PointsReader in;
+ private final int maxDoc;
+
+ AssertingPointsReader(int maxDoc, PointsReader in) {
+ this.in = in;
+ this.maxDoc = maxDoc;
+ // do a few simple checks on init
+ assert toString() != null;
+ assert ramBytesUsed() >= 0;
+ assert getChildResources() != null;
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ in.close(); // close again
+ }
+
+ @Override
+ public void intersect(String fieldName, IntersectVisitor visitor) throws IOException {
+ in.intersect(fieldName,
+ new AssertingIntersectVisitor(in.getNumDimensions(fieldName), in.getBytesPerDimension(fieldName), visitor));
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ long v = in.ramBytesUsed();
+ assert v >= 0;
+ return v;
+ }
+
+ @Override
+ public Collection<Accountable> getChildResources() {
+ Collection<Accountable> res = in.getChildResources();
+ TestUtil.checkReadOnly(res);
+ return res;
+ }
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ in.checkIntegrity();
+ }
+
+ @Override
+ public PointsReader getMergeInstance() throws IOException {
+ return new AssertingPointsReader(maxDoc, in.getMergeInstance());
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(" + in.toString() + ")";
+ }
+
+ @Override
+ public byte[] getMinPackedValue(String fieldName) throws IOException {
+ assertStats(fieldName);
+ return in.getMinPackedValue(fieldName);
+ }
+
+ @Override
+ public byte[] getMaxPackedValue(String fieldName) throws IOException {
+ assertStats(fieldName);
+ return in.getMaxPackedValue(fieldName);
+ }
+
+ @Override
+ public int getNumDimensions(String fieldName) throws IOException {
+ assertStats(fieldName);
+ return in.getNumDimensions(fieldName);
+ }
+
+ @Override
+ public int getBytesPerDimension(String fieldName) throws IOException {
+ assertStats(fieldName);
+ return in.getBytesPerDimension(fieldName);
+ }
+
+ @Override
+ public long size(String fieldName) {
+ assertStats(fieldName);
+ return in.size(fieldName);
+ }
+
+ @Override
+ public int getDocCount(String fieldName) {
+ assertStats(fieldName);
+ return in.getDocCount(fieldName);
+ }
+
+ private void assertStats(String fieldName) {
+ assert in.size(fieldName) >= 0;
+ assert in.getDocCount(fieldName) >= 0;
+ assert in.getDocCount(fieldName) <= in.size(fieldName);
+ assert in.getDocCount(fieldName) <= maxDoc;
+ }
+ }
+
+ static class AssertingPointsWriter extends PointsWriter {
+ private final PointsWriter in;
+
+ AssertingPointsWriter(SegmentWriteState writeState, PointsWriter in) {
+ this.in = in;
+ }
+
+ @Override
+ public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException {
+ if (fieldInfo.getPointDimensionCount() == 0) {
+ throw new IllegalArgumentException("writing field=\"" + fieldInfo.name + "\" but pointDimensionalCount is 0");
+ }
+ in.writeField(fieldInfo, values);
+ }
+
+ @Override
+ public void merge(MergeState mergeState) throws IOException {
+ in.merge(mergeState);
+ }
+
+ @Override
+ public void finish() throws IOException {
+ in.finish();
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ in.close(); // close again
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74e7e150/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCodec.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCodec.java
index 314ff95..086a574 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCodec.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCodec.java
@@ -25,7 +25,7 @@ import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.PointFormat;
+import org.apache.lucene.codecs.PointsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
@@ -92,8 +92,8 @@ public class CrankyCodec extends FilterCodec {
}
@Override
- public PointFormat pointFormat() {
- return new CrankyPointFormat(delegate.pointFormat(), random);
+ public PointsFormat pointsFormat() {
+ return new CrankyPointsFormat(delegate.pointsFormat(), random);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74e7e150/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointFormat.java
deleted file mode 100644
index b663a80..0000000
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointFormat.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.codecs.cranky;
-
-import java.io.IOException;
-import java.util.Random;
-
-import org.apache.lucene.codecs.PointFormat;
-import org.apache.lucene.codecs.PointReader;
-import org.apache.lucene.codecs.PointWriter;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.MergeState;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-
-class CrankyPointFormat extends PointFormat {
- PointFormat delegate;
- Random random;
-
- CrankyPointFormat(PointFormat delegate, Random random) {
- this.delegate = delegate;
- this.random = random;
- }
-
- @Override
- public PointWriter fieldsWriter(SegmentWriteState state) throws IOException {
- return new CrankyPointWriter(delegate.fieldsWriter(state), random);
- }
-
- @Override
- public PointReader fieldsReader(SegmentReadState state) throws IOException {
- return new CrankyPointReader(delegate.fieldsReader(state), random);
- }
-
- static class CrankyPointWriter extends PointWriter {
- final PointWriter delegate;
- final Random random;
-
- public CrankyPointWriter(PointWriter delegate, Random random) {
- this.delegate = delegate;
- this.random = random;
- }
-
- @Override
- public void writeField(FieldInfo fieldInfo, PointReader values) throws IOException {
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- delegate.writeField(fieldInfo, values);
- }
-
- @Override
- public void finish() throws IOException {
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- delegate.finish();
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- }
-
- @Override
- public void merge(MergeState mergeState) throws IOException {
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- delegate.merge(mergeState);
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- }
-
- @Override
- public void close() throws IOException {
- delegate.close();
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- }
- }
-
- static class CrankyPointReader extends PointReader {
- final PointReader delegate;
- final Random random;
- public CrankyPointReader(PointReader delegate, Random random) {
- this.delegate = delegate;
- this.random = random;
- }
-
- @Override
- public void checkIntegrity() throws IOException {
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- delegate.checkIntegrity();
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- }
-
- @Override
- public void intersect(String fieldName, IntersectVisitor visitor) throws IOException {
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- delegate.intersect(fieldName, visitor);
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- }
-
- @Override
- public byte[] getMinPackedValue(String fieldName) throws IOException {
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- return delegate.getMinPackedValue(fieldName);
- }
-
- @Override
- public byte[] getMaxPackedValue(String fieldName) throws IOException {
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- return delegate.getMaxPackedValue(fieldName);
- }
-
- @Override
- public int getNumDimensions(String fieldName) throws IOException {
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- return delegate.getNumDimensions(fieldName);
- }
-
- @Override
- public int getBytesPerDimension(String fieldName) throws IOException {
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- return delegate.getBytesPerDimension(fieldName);
- }
-
- @Override
- public void close() throws IOException {
- delegate.close();
- if (random.nextInt(100) == 0) {
- throw new IOException("Fake IOException");
- }
- }
-
- @Override
- public long ramBytesUsed() {
- return delegate.ramBytesUsed();
- }
-
- @Override
- public long size(String fieldName) {
- return delegate.size(fieldName);
- }
-
- @Override
- public int getDocCount(String fieldName) {
- return delegate.getDocCount(fieldName);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74e7e150/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java
new file mode 100644
index 0000000..4d16461
--- /dev/null
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.cranky;
+
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PointsReader;
+import org.apache.lucene.codecs.PointsWriter;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.MergeState;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+
+class CrankyPointsFormat extends PointsFormat {
+ PointsFormat delegate;
+ Random random;
+
+ CrankyPointsFormat(PointsFormat delegate, Random random) {
+ this.delegate = delegate;
+ this.random = random;
+ }
+
+ @Override
+ public PointsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+ return new CrankyPointsWriter(delegate.fieldsWriter(state), random);
+ }
+
+ @Override
+ public PointsReader fieldsReader(SegmentReadState state) throws IOException {
+ return new CrankyPointsReader(delegate.fieldsReader(state), random);
+ }
+
+ static class CrankyPointsWriter extends PointsWriter {
+ final PointsWriter delegate;
+ final Random random;
+
+ public CrankyPointsWriter(PointsWriter delegate, Random random) {
+ this.delegate = delegate;
+ this.random = random;
+ }
+
+ @Override
+ public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException {
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ delegate.writeField(fieldInfo, values);
+ }
+
+ @Override
+ public void finish() throws IOException {
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ delegate.finish();
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ }
+
+ @Override
+ public void merge(MergeState mergeState) throws IOException {
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ delegate.merge(mergeState);
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ }
+ }
+
+ static class CrankyPointsReader extends PointsReader {
+ final PointsReader delegate;
+ final Random random;
+ public CrankyPointsReader(PointsReader delegate, Random random) {
+ this.delegate = delegate;
+ this.random = random;
+ }
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ delegate.checkIntegrity();
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ }
+
+ @Override
+ public void intersect(String fieldName, IntersectVisitor visitor) throws IOException {
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ delegate.intersect(fieldName, visitor);
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ }
+
+ @Override
+ public byte[] getMinPackedValue(String fieldName) throws IOException {
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ return delegate.getMinPackedValue(fieldName);
+ }
+
+ @Override
+ public byte[] getMaxPackedValue(String fieldName) throws IOException {
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ return delegate.getMaxPackedValue(fieldName);
+ }
+
+ @Override
+ public int getNumDimensions(String fieldName) throws IOException {
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ return delegate.getNumDimensions(fieldName);
+ }
+
+ @Override
+ public int getBytesPerDimension(String fieldName) throws IOException {
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ return delegate.getBytesPerDimension(fieldName);
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ return delegate.ramBytesUsed();
+ }
+
+ @Override
+ public long size(String fieldName) {
+ return delegate.size(fieldName);
+ }
+
+ @Override
+ public int getDocCount(String fieldName) {
+ return delegate.getDocCount(fieldName);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74e7e150/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java
deleted file mode 100644
index 16a6b6b..0000000
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java
+++ /dev/null
@@ -1,950 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.index;
-
-import java.io.IOException;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.List;
-
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.document.BinaryPoint;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.IntPoint;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.PointValues.IntersectVisitor;
-import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.NumericUtils;
-import org.apache.lucene.util.StringHelper;
-import org.apache.lucene.util.TestUtil;
-
-/**
- * Abstract class to do basic tests for a points format.
- * NOTE: This test focuses on the points impl, nothing else.
- * The [stretch] goal is for this test to be
- * so thorough in testing a new PointFormat that if this
- * test passes, then all Lucene/Solr tests should also pass. Ie,
- * if there is some bug in a given PointFormat that this
- * test fails to catch then this test needs to be improved! */
-public abstract class BasePointFormatTestCase extends BaseIndexFileFormatTestCase {
-
- @Override
- protected void addRandomFields(Document doc) {
- final int numValues = random().nextInt(3);
- for (int i = 0; i < numValues; i++) {
- doc.add(new IntPoint("f", random().nextInt()));
- }
- }
-
- public void testBasic() throws Exception {
- Directory dir = getDirectory(20);
- IndexWriterConfig iwc = newIndexWriterConfig();
- iwc.setMergePolicy(newLogMergePolicy());
- IndexWriter w = new IndexWriter(dir, iwc);
- byte[] point = new byte[4];
- for(int i=0;i<20;i++) {
- Document doc = new Document();
- NumericUtils.intToSortableBytes(i, point, 0);
- doc.add(new BinaryPoint("dim", point));
- w.addDocument(doc);
- }
- w.forceMerge(1);
- w.close();
-
- DirectoryReader r = DirectoryReader.open(dir);
- LeafReader sub = getOnlySegmentReader(r);
- PointValues values = sub.getPointValues();
-
- // Simple test: make sure intersect can visit every doc:
- BitSet seen = new BitSet();
- values.intersect("dim",
- new IntersectVisitor() {
- @Override
- public Relation compare(byte[] minPacked, byte[] maxPacked) {
- return Relation.CELL_CROSSES_QUERY;
- }
- public void visit(int docID) {
- throw new IllegalStateException();
- }
- public void visit(int docID, byte[] packedValue) {
- seen.set(docID);
- assertEquals(docID, NumericUtils.sortableBytesToInt(packedValue, 0));
- }
- });
- assertEquals(20, seen.cardinality());
- IOUtils.close(r, dir);
- }
-
- public void testMerge() throws Exception {
- Directory dir = getDirectory(20);
- IndexWriterConfig iwc = newIndexWriterConfig();
- iwc.setMergePolicy(newLogMergePolicy());
- IndexWriter w = new IndexWriter(dir, iwc);
- byte[] point = new byte[4];
- for(int i=0;i<20;i++) {
- Document doc = new Document();
- NumericUtils.intToSortableBytes(i, point, 0);
- doc.add(new BinaryPoint("dim", point));
- w.addDocument(doc);
- if (i == 10) {
- w.commit();
- }
- }
- w.forceMerge(1);
- w.close();
-
- DirectoryReader r = DirectoryReader.open(dir);
- LeafReader sub = getOnlySegmentReader(r);
- PointValues values = sub.getPointValues();
-
- // Simple test: make sure intersect can visit every doc:
- BitSet seen = new BitSet();
- values.intersect("dim",
- new IntersectVisitor() {
- @Override
- public Relation compare(byte[] minPacked, byte[] maxPacked) {
- return Relation.CELL_CROSSES_QUERY;
- }
- public void visit(int docID) {
- throw new IllegalStateException();
- }
- public void visit(int docID, byte[] packedValue) {
- seen.set(docID);
- assertEquals(docID, NumericUtils.sortableBytesToInt(packedValue, 0));
- }
- });
- assertEquals(20, seen.cardinality());
- IOUtils.close(r, dir);
- }
-
- public void testAllPointDocsDeletedInSegment() throws Exception {
- Directory dir = getDirectory(20);
- IndexWriterConfig iwc = newIndexWriterConfig();
- IndexWriter w = new IndexWriter(dir, iwc);
- byte[] point = new byte[4];
- for(int i=0;i<10;i++) {
- Document doc = new Document();
- NumericUtils.intToSortableBytes(i, point, 0);
- doc.add(new BinaryPoint("dim", point));
- doc.add(new NumericDocValuesField("id", i));
- doc.add(newStringField("x", "x", Field.Store.NO));
- w.addDocument(doc);
- }
- w.addDocument(new Document());
- w.deleteDocuments(new Term("x", "x"));
- if (random().nextBoolean()) {
- w.forceMerge(1);
- }
- w.close();
- DirectoryReader r = DirectoryReader.open(dir);
- assertEquals(1, r.numDocs());
- Bits liveDocs = MultiFields.getLiveDocs(r);
-
- for(LeafReaderContext ctx : r.leaves()) {
- PointValues values = ctx.reader().getPointValues();
- NumericDocValues idValues = ctx.reader().getNumericDocValues("id");
- if (values != null) {
- BitSet seen = new BitSet();
- values.intersect("dim",
- new IntersectVisitor() {
- @Override
- public Relation compare(byte[] minPacked, byte[] maxPacked) {
- return Relation.CELL_CROSSES_QUERY;
- }
- public void visit(int docID) {
- throw new IllegalStateException();
- }
- public void visit(int docID, byte[] packedValue) {
- if (liveDocs.get(docID)) {
- seen.set(docID);
- }
- assertEquals(idValues.get(docID), NumericUtils.sortableBytesToInt(packedValue, 0));
- }
- });
- assertEquals(0, seen.cardinality());
- }
- }
- IOUtils.close(r, dir);
- }
-
- /** Make sure we close open files, delete temp files, etc., on exception */
- public void testWithExceptions() throws Exception {
- int numDocs = atLeast(10000);
- int numBytesPerDim = TestUtil.nextInt(random(), 2, PointValues.MAX_NUM_BYTES);
- int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS);
-
- byte[][][] docValues = new byte[numDocs][][];
-
- for(int docID=0;docID<numDocs;docID++) {
- byte[][] values = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
- values[dim] = new byte[numBytesPerDim];
- random().nextBytes(values[dim]);
- }
- docValues[docID] = values;
- }
-
- // Keep retrying until we 1) we allow a big enough heap, and 2) we hit a random IOExc from MDW:
- boolean done = false;
- while (done == false) {
- try (MockDirectoryWrapper dir = newMockFSDirectory(createTempDir())) {
- try {
- dir.setRandomIOExceptionRate(0.05);
- dir.setRandomIOExceptionRateOnOpen(0.05);
- verify(dir, docValues, null, numDims, numBytesPerDim, true);
- } catch (IllegalStateException ise) {
- if (ise.getMessage().contains("this writer hit an unrecoverable error")) {
- Throwable cause = ise.getCause();
- if (cause != null && cause.getMessage().contains("a random IOException")) {
- done = true;
- } else {
- throw ise;
- }
- } else {
- throw ise;
- }
- } catch (AssertionError ae) {
- if (ae.getMessage().contains("does not exist; files=")) {
- // OK: likely we threw the random IOExc when IW was asserting the commit files exist
- done = true;
- } else {
- throw ae;
- }
- } catch (IllegalArgumentException iae) {
- // This just means we got a too-small maxMB for the maxPointsInLeafNode; just retry w/ more heap
- assertTrue(iae.getMessage().contains("either increase maxMBSortInHeap or decrease maxPointsInLeafNode"));
- } catch (IOException ioe) {
- String message = ioe.getMessage();
- if (message.contains("a random IOException") || message.contains("background merge hit exception")) {
- // BKDWriter should fully clean up after itself:
- done = true;
- } else {
- throw ioe;
- }
- }
- }
- }
- }
-
- public void testMultiValued() throws Exception {
- int numBytesPerDim = TestUtil.nextInt(random(), 2, PointValues.MAX_NUM_BYTES);
- int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS);
-
- int numDocs = atLeast(1000);
- List<byte[][]> docValues = new ArrayList<>();
- List<Integer> docIDs = new ArrayList<>();
-
- for(int docID=0;docID<numDocs;docID++) {
- int numValuesInDoc = TestUtil.nextInt(random(), 1, 5);
- for(int ord=0;ord<numValuesInDoc;ord++) {
- docIDs.add(docID);
- byte[][] values = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
- values[dim] = new byte[numBytesPerDim];
- random().nextBytes(values[dim]);
- }
- docValues.add(values);
- }
- }
-
- byte[][][] docValuesArray = docValues.toArray(new byte[docValues.size()][][]);
- int[] docIDsArray = new int[docIDs.size()];
- for(int i=0;i<docIDsArray.length;i++) {
- docIDsArray[i] = docIDs.get(i);
- }
-
- verify(docValuesArray, docIDsArray, numDims, numBytesPerDim);
- }
-
- public void testAllEqual() throws Exception {
- int numBytesPerDim = TestUtil.nextInt(random(), 2, PointValues.MAX_NUM_BYTES);
- int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS);
-
- int numDocs = atLeast(1000);
- byte[][][] docValues = new byte[numDocs][][];
-
- for(int docID=0;docID<numDocs;docID++) {
- if (docID == 0) {
- byte[][] values = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
- values[dim] = new byte[numBytesPerDim];
- random().nextBytes(values[dim]);
- }
- docValues[docID] = values;
- } else {
- docValues[docID] = docValues[0];
- }
- }
-
- verify(docValues, null, numDims, numBytesPerDim);
- }
-
- public void testOneDimEqual() throws Exception {
- int numBytesPerDim = TestUtil.nextInt(random(), 2, PointValues.MAX_NUM_BYTES);
- int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS);
-
- int numDocs = atLeast(1000);
- int theEqualDim = random().nextInt(numDims);
- byte[][][] docValues = new byte[numDocs][][];
-
- for(int docID=0;docID<numDocs;docID++) {
- byte[][] values = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
- values[dim] = new byte[numBytesPerDim];
- random().nextBytes(values[dim]);
- }
- docValues[docID] = values;
- if (docID > 0) {
- docValues[docID][theEqualDim] = docValues[0][theEqualDim];
- }
- }
-
- verify(docValues, null, numDims, numBytesPerDim);
- }
-
- // Tests on N-dimensional points where each dimension is a BigInteger
- public void testBigIntNDims() throws Exception {
-
- int numDocs = atLeast(1000);
- try (Directory dir = getDirectory(numDocs)) {
- int numBytesPerDim = TestUtil.nextInt(random(), 2, PointValues.MAX_NUM_BYTES);
- int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS);
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- // We rely on docIDs not changing:
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
- BigInteger[][] docs = new BigInteger[numDocs][];
-
- for(int docID=0;docID<numDocs;docID++) {
- BigInteger[] values = new BigInteger[numDims];
- if (VERBOSE) {
- System.out.println(" docID=" + docID);
- }
- byte[][] bytes = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
- values[dim] = randomBigInt(numBytesPerDim);
- bytes[dim] = new byte[numBytesPerDim];
- NumericUtils.bigIntToSortableBytes(values[dim], numBytesPerDim, bytes[dim], 0);
- if (VERBOSE) {
- System.out.println(" " + dim + " -> " + values[dim]);
- }
- }
- docs[docID] = values;
- Document doc = new Document();
- doc.add(new BinaryPoint("field", bytes));
- w.addDocument(doc);
- }
-
- DirectoryReader r = w.getReader();
- w.close();
-
- int iters = atLeast(100);
- for(int iter=0;iter<iters;iter++) {
- if (VERBOSE) {
- System.out.println("\nTEST: iter=" + iter);
- }
-
- // Random N dims rect query:
- BigInteger[] queryMin = new BigInteger[numDims];
- BigInteger[] queryMax = new BigInteger[numDims];
- for(int dim=0;dim<numDims;dim++) {
- queryMin[dim] = randomBigInt(numBytesPerDim);
- queryMax[dim] = randomBigInt(numBytesPerDim);
- if (queryMin[dim].compareTo(queryMax[dim]) > 0) {
- BigInteger x = queryMin[dim];
- queryMin[dim] = queryMax[dim];
- queryMax[dim] = x;
- }
- if (VERBOSE) {
- System.out.println(" " + dim + "\n min=" + queryMin[dim] + "\n max=" + queryMax[dim]);
- }
- }
-
- final BitSet hits = new BitSet();
- for(LeafReaderContext ctx : r.leaves()) {
- PointValues dimValues = ctx.reader().getPointValues();
- if (dimValues == null) {
- continue;
- }
-
- final int docBase = ctx.docBase;
-
- dimValues.intersect("field", new IntersectVisitor() {
- @Override
- public void visit(int docID) {
- hits.set(docBase+docID);
- //System.out.println("visit docID=" + docID);
- }
-
- @Override
- public void visit(int docID, byte[] packedValue) {
- //System.out.println("visit check docID=" + docID);
- for(int dim=0;dim<numDims;dim++) {
- BigInteger x = NumericUtils.sortableBytesToBigInt(packedValue, dim * numBytesPerDim, numBytesPerDim);
- if (x.compareTo(queryMin[dim]) < 0 || x.compareTo(queryMax[dim]) > 0) {
- //System.out.println(" no");
- return;
- }
- }
-
- //System.out.println(" yes");
- hits.set(docBase+docID);
- }
-
- @Override
- public Relation compare(byte[] minPacked, byte[] maxPacked) {
- boolean crosses = false;
- for(int dim=0;dim<numDims;dim++) {
- BigInteger min = NumericUtils.sortableBytesToBigInt(minPacked, dim * numBytesPerDim, numBytesPerDim);
- BigInteger max = NumericUtils.sortableBytesToBigInt(maxPacked, dim * numBytesPerDim, numBytesPerDim);
- assert max.compareTo(min) >= 0;
-
- if (max.compareTo(queryMin[dim]) < 0 || min.compareTo(queryMax[dim]) > 0) {
- return Relation.CELL_OUTSIDE_QUERY;
- } else if (min.compareTo(queryMin[dim]) < 0 || max.compareTo(queryMax[dim]) > 0) {
- crosses = true;
- }
- }
-
- if (crosses) {
- return Relation.CELL_CROSSES_QUERY;
- } else {
- return Relation.CELL_INSIDE_QUERY;
- }
- }
- });
- }
-
- for(int docID=0;docID<numDocs;docID++) {
- BigInteger[] docValues = docs[docID];
- boolean expected = true;
- for(int dim=0;dim<numDims;dim++) {
- BigInteger x = docValues[dim];
- if (x.compareTo(queryMin[dim]) < 0 || x.compareTo(queryMax[dim]) > 0) {
- expected = false;
- break;
- }
- }
- boolean actual = hits.get(docID);
- assertEquals("docID=" + docID, expected, actual);
- }
- }
- r.close();
- }
- }
-
- public void testRandomBinaryTiny() throws Exception {
- doTestRandomBinary(10);
- }
-
- public void testRandomBinaryMedium() throws Exception {
- doTestRandomBinary(10000);
- }
-
- @Nightly
- public void testRandomBinaryBig() throws Exception {
- assumeFalse("too slow with SimpleText", Codec.getDefault().getName().equals("SimpleText"));
- doTestRandomBinary(200000);
- }
-
- private void doTestRandomBinary(int count) throws Exception {
- int numDocs = TestUtil.nextInt(random(), count, count*2);
- int numBytesPerDim = TestUtil.nextInt(random(), 2, PointValues.MAX_NUM_BYTES);
- int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS);
-
- byte[][][] docValues = new byte[numDocs][][];
-
- for(int docID=0;docID<numDocs;docID++) {
- byte[][] values = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
- values[dim] = new byte[numBytesPerDim];
- // TODO: sometimes test on a "small" volume too, so we test the high density cases, higher chance of boundary, etc. cases:
- random().nextBytes(values[dim]);
- }
- docValues[docID] = values;
- }
-
- verify(docValues, null, numDims, numBytesPerDim);
- }
-
- /** docIDs can be null, for the single valued case, else it maps value to docID, but all values for one doc must be adjacent */
- private void verify(byte[][][] docValues, int[] docIDs, int numDims, int numBytesPerDim) throws Exception {
- try (Directory dir = getDirectory(docValues.length)) {
- while (true) {
- try {
- verify(dir, docValues, docIDs, numDims, numBytesPerDim, false);
- return;
- } catch (IllegalArgumentException iae) {
- // This just means we got a too-small maxMB for the maxPointsInLeafNode; just retry
- assertTrue(iae.getMessage().contains("either increase maxMBSortInHeap or decrease maxPointsInLeafNode"));
- }
- }
- }
- }
-
- private void verify(Directory dir, byte[][][] docValues, int[] ids, int numDims, int numBytesPerDim, boolean expectExceptions) throws Exception {
- int numValues = docValues.length;
- if (VERBOSE) {
- System.out.println("TEST: numValues=" + numValues + " numDims=" + numDims + " numBytesPerDim=" + numBytesPerDim);
- }
-
- // RandomIndexWriter is too slow:
- boolean useRealWriter = docValues.length > 10000;
-
- IndexWriterConfig iwc;
- if (useRealWriter) {
- iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- } else {
- iwc = newIndexWriterConfig();
- }
-
- if (expectExceptions) {
- MergeScheduler ms = iwc.getMergeScheduler();
- if (ms instanceof ConcurrentMergeScheduler) {
- ((ConcurrentMergeScheduler) ms).setSuppressExceptions();
- }
- }
- RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
- DirectoryReader r = null;
-
- // Compute actual min/max values:
- byte[][] expectedMinValues = new byte[numDims][];
- byte[][] expectedMaxValues = new byte[numDims][];
- for(int ord=0;ord<docValues.length;ord++) {
- for(int dim=0;dim<numDims;dim++) {
- if (ord == 0) {
- expectedMinValues[dim] = new byte[numBytesPerDim];
- System.arraycopy(docValues[ord][dim], 0, expectedMinValues[dim], 0, numBytesPerDim);
- expectedMaxValues[dim] = new byte[numBytesPerDim];
- System.arraycopy(docValues[ord][dim], 0, expectedMaxValues[dim], 0, numBytesPerDim);
- } else {
- // TODO: it's cheating that we use StringHelper.compare for "truth": what if it's buggy?
- if (StringHelper.compare(numBytesPerDim, docValues[ord][dim], 0, expectedMinValues[dim], 0) < 0) {
- System.arraycopy(docValues[ord][dim], 0, expectedMinValues[dim], 0, numBytesPerDim);
- }
- if (StringHelper.compare(numBytesPerDim, docValues[ord][dim], 0, expectedMaxValues[dim], 0) > 0) {
- System.arraycopy(docValues[ord][dim], 0, expectedMaxValues[dim], 0, numBytesPerDim);
- }
- }
- }
- }
-
- // 20% of the time we add into a separate directory, then at some point use
- // addIndexes to bring the indexed point values to the main directory:
- Directory saveDir;
- RandomIndexWriter saveW;
- int addIndexesAt;
- if (random().nextInt(5) == 1) {
- saveDir = dir;
- saveW = w;
- dir = getDirectory(numValues);
- if (useRealWriter) {
- iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- } else {
- iwc = newIndexWriterConfig();
- }
- if (expectExceptions) {
- MergeScheduler ms = iwc.getMergeScheduler();
- if (ms instanceof ConcurrentMergeScheduler) {
- ((ConcurrentMergeScheduler) ms).setSuppressExceptions();
- }
- }
- w = new RandomIndexWriter(random(), dir, iwc);
- addIndexesAt = TestUtil.nextInt(random(), 1, numValues-1);
- } else {
- saveW = null;
- saveDir = null;
- addIndexesAt = 0;
- }
-
- try {
-
- Document doc = null;
- int lastID = -1;
- for(int ord=0;ord<numValues;ord++) {
- int id;
- if (ids == null) {
- id = ord;
- } else {
- id = ids[ord];
- }
- if (id != lastID) {
- if (doc != null) {
- if (useRealWriter) {
- w.w.addDocument(doc);
- } else {
- w.addDocument(doc);
- }
- }
- doc = new Document();
- doc.add(new NumericDocValuesField("id", id));
- }
- doc.add(new BinaryPoint("field", docValues[ord]));
- lastID = id;
-
- if (random().nextInt(30) == 17) {
- // randomly index some documents without this field
- if (useRealWriter) {
- w.w.addDocument(new Document());
- } else {
- w.addDocument(new Document());
- }
- if (VERBOSE) {
- System.out.println("add empty doc");
- }
- }
-
- if (random().nextInt(30) == 17) {
- // randomly index some documents with this field, but we will delete them:
- Document xdoc = new Document();
- xdoc.add(new BinaryPoint("field", docValues[ord]));
- xdoc.add(new StringField("nukeme", "yes", Field.Store.NO));
- if (useRealWriter) {
- w.w.addDocument(xdoc);
- } else {
- w.addDocument(xdoc);
- }
- if (VERBOSE) {
- System.out.println("add doc doc-to-delete");
- }
-
- if (random().nextInt(5) == 1) {
- if (useRealWriter) {
- w.w.deleteDocuments(new Term("nukeme", "yes"));
- } else {
- w.deleteDocuments(new Term("nukeme", "yes"));
- }
- }
- }
-
- if (VERBOSE) {
- System.out.println(" ord=" + ord + " id=" + id);
- for(int dim=0;dim<numDims;dim++) {
- System.out.println(" dim=" + dim + " value=" + new BytesRef(docValues[ord][dim]));
- }
- }
-
- if (saveW != null && ord >= addIndexesAt) {
- switchIndex(w, dir, saveW);
- w = saveW;
- dir = saveDir;
- saveW = null;
- saveDir = null;
- }
- }
- w.addDocument(doc);
- w.deleteDocuments(new Term("nukeme", "yes"));
-
- if (random().nextBoolean()) {
- if (VERBOSE) {
- System.out.println("\nTEST: now force merge");
- }
- w.forceMerge(1);
- }
-
- r = w.getReader();
- w.close();
-
- if (VERBOSE) {
- System.out.println("TEST: reader=" + r);
- }
-
- NumericDocValues idValues = MultiDocValues.getNumericValues(r, "id");
- Bits liveDocs = MultiFields.getLiveDocs(r);
-
- // Verify min/max values are correct:
- byte[] minValues = new byte[numDims*numBytesPerDim];
- Arrays.fill(minValues, (byte) 0xff);
-
- byte[] maxValues = new byte[numDims*numBytesPerDim];
-
- for(LeafReaderContext ctx : r.leaves()) {
- PointValues dimValues = ctx.reader().getPointValues();
- if (dimValues == null) {
- continue;
- }
-
- byte[] leafMinValues = dimValues.getMinPackedValue("field");
- byte[] leafMaxValues = dimValues.getMaxPackedValue("field");
- for(int dim=0;dim<numDims;dim++) {
- if (StringHelper.compare(numBytesPerDim, leafMinValues, dim*numBytesPerDim, minValues, dim*numBytesPerDim) < 0) {
- System.arraycopy(leafMinValues, dim*numBytesPerDim, minValues, dim*numBytesPerDim, numBytesPerDim);
- }
- if (StringHelper.compare(numBytesPerDim, leafMaxValues, dim*numBytesPerDim, maxValues, dim*numBytesPerDim) > 0) {
- System.arraycopy(leafMaxValues, dim*numBytesPerDim, maxValues, dim*numBytesPerDim, numBytesPerDim);
- }
- }
- }
-
- byte[] scratch = new byte[numBytesPerDim];
- for(int dim=0;dim<numDims;dim++) {
- System.arraycopy(minValues, dim*numBytesPerDim, scratch, 0, numBytesPerDim);
- //System.out.println("dim=" + dim + " expectedMin=" + new BytesRef(expectedMinValues[dim]) + " min=" + new BytesRef(scratch));
- assertTrue(Arrays.equals(expectedMinValues[dim], scratch));
- System.arraycopy(maxValues, dim*numBytesPerDim, scratch, 0, numBytesPerDim);
- //System.out.println("dim=" + dim + " expectedMax=" + new BytesRef(expectedMaxValues[dim]) + " max=" + new BytesRef(scratch));
- assertTrue(Arrays.equals(expectedMaxValues[dim], scratch));
- }
-
- int iters = atLeast(100);
- for(int iter=0;iter<iters;iter++) {
- if (VERBOSE) {
- System.out.println("\nTEST: iter=" + iter);
- }
-
- // Random N dims rect query:
- byte[][] queryMin = new byte[numDims][];
- byte[][] queryMax = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
- queryMin[dim] = new byte[numBytesPerDim];
- random().nextBytes(queryMin[dim]);
- queryMax[dim] = new byte[numBytesPerDim];
- random().nextBytes(queryMax[dim]);
- if (StringHelper.compare(numBytesPerDim, queryMin[dim], 0, queryMax[dim], 0) > 0) {
- byte[] x = queryMin[dim];
- queryMin[dim] = queryMax[dim];
- queryMax[dim] = x;
- }
- }
-
- if (VERBOSE) {
- for(int dim=0;dim<numDims;dim++) {
- System.out.println(" dim=" + dim + "\n queryMin=" + new BytesRef(queryMin[dim]) + "\n queryMax=" + new BytesRef(queryMax[dim]));
- }
- }
-
- final BitSet hits = new BitSet();
-
- for(LeafReaderContext ctx : r.leaves()) {
- PointValues dimValues = ctx.reader().getPointValues();
- if (dimValues == null) {
- continue;
- }
-
- final int docBase = ctx.docBase;
-
- dimValues.intersect("field", new PointValues.IntersectVisitor() {
- @Override
- public void visit(int docID) {
- if (liveDocs == null || liveDocs.get(docBase+docID)) {
- hits.set((int) idValues.get(docBase+docID));
- }
- //System.out.println("visit docID=" + docID);
- }
-
- @Override
- public void visit(int docID, byte[] packedValue) {
- if (liveDocs != null && liveDocs.get(docBase+docID) == false) {
- return;
- }
-
- //System.out.println("visit check docID=" + docID + " id=" + idValues.get(docID));
- for(int dim=0;dim<numDims;dim++) {
- //System.out.println(" dim=" + dim + " value=" + new BytesRef(packedValue, dim*numBytesPerDim, numBytesPerDim));
- if (StringHelper.compare(numBytesPerDim, packedValue, dim*numBytesPerDim, queryMin[dim], 0) < 0 ||
- StringHelper.compare(numBytesPerDim, packedValue, dim*numBytesPerDim, queryMax[dim], 0) > 0) {
- //System.out.println(" no");
- return;
- }
- }
-
- //System.out.println(" yes");
- hits.set((int) idValues.get(docBase+docID));
- }
-
- @Override
- public Relation compare(byte[] minPacked, byte[] maxPacked) {
- boolean crosses = false;
- //System.out.println("compare");
- for(int dim=0;dim<numDims;dim++) {
- if (StringHelper.compare(numBytesPerDim, maxPacked, dim*numBytesPerDim, queryMin[dim], 0) < 0 ||
- StringHelper.compare(numBytesPerDim, minPacked, dim*numBytesPerDim, queryMax[dim], 0) > 0) {
- //System.out.println(" query_outside_cell");
- return Relation.CELL_OUTSIDE_QUERY;
- } else if (StringHelper.compare(numBytesPerDim, minPacked, dim*numBytesPerDim, queryMin[dim], 0) < 0 ||
- StringHelper.compare(numBytesPerDim, maxPacked, dim*numBytesPerDim, queryMax[dim], 0) > 0) {
- crosses = true;
- }
- }
-
- if (crosses) {
- //System.out.println(" query_crosses_cell");
- return Relation.CELL_CROSSES_QUERY;
- } else {
- //System.out.println(" cell_inside_query");
- return Relation.CELL_INSIDE_QUERY;
- }
- }
- });
- }
-
- BitSet expected = new BitSet();
- for(int ord=0;ord<numValues;ord++) {
- boolean matches = true;
- for(int dim=0;dim<numDims;dim++) {
- byte[] x = docValues[ord][dim];
- if (StringHelper.compare(numBytesPerDim, x, 0, queryMin[dim], 0) < 0 ||
- StringHelper.compare(numBytesPerDim, x, 0, queryMax[dim], 0) > 0) {
- matches = false;
- break;
- }
- }
-
- if (matches) {
- int id;
- if (ids == null) {
- id = ord;
- } else {
- id = ids[ord];
- }
- expected.set(id);
- }
- }
-
- int limit = Math.max(expected.length(), hits.length());
- int failCount = 0;
- int successCount = 0;
- for(int id=0;id<limit;id++) {
- if (expected.get(id) != hits.get(id)) {
- System.out.println("FAIL: id=" + id);
- failCount++;
- } else {
- successCount++;
- }
- }
-
- if (failCount != 0) {
- for(int docID=0;docID<r.maxDoc();docID++) {
- System.out.println(" docID=" + docID + " id=" + idValues.get(docID));
- }
-
- fail(failCount + " docs failed; " + successCount + " docs succeeded");
- }
- }
- } finally {
- IOUtils.closeWhileHandlingException(r, w, saveW, saveDir == null ? null : dir);
- }
- }
-
- public void testAddIndexes() throws IOException {
- Directory dir1 = newDirectory();
- RandomIndexWriter w = new RandomIndexWriter(random(), dir1);
- Document doc = new Document();
- doc.add(new IntPoint("int1", 17));
- w.addDocument(doc);
- doc = new Document();
- doc.add(new IntPoint("int2", 42));
- w.addDocument(doc);
- w.close();
-
- // Different field number assigments:
- Directory dir2 = newDirectory();
- w = new RandomIndexWriter(random(), dir2);
- doc = new Document();
- doc.add(new IntPoint("int2", 42));
- w.addDocument(doc);
- doc = new Document();
- doc.add(new IntPoint("int1", 17));
- w.addDocument(doc);
- w.close();
-
- Directory dir = newDirectory();
- w = new RandomIndexWriter(random(), dir);
- w.addIndexes(new Directory[] {dir1, dir2});
- w.forceMerge(1);
-
- DirectoryReader r = w.getReader();
- IndexSearcher s = newSearcher(r, false);
- assertEquals(2, s.count(IntPoint.newExactQuery("int1", 17)));
- assertEquals(2, s.count(IntPoint.newExactQuery("int2", 42)));
- r.close();
- w.close();
- dir.close();
- dir1.close();
- dir2.close();
- }
-
- private void switchIndex(RandomIndexWriter w, Directory dir, RandomIndexWriter saveW) throws IOException {
- if (random().nextBoolean()) {
- // Add via readers:
- try (DirectoryReader r = w.getReader()) {
- if (random().nextBoolean()) {
- // Add via CodecReaders:
- List<CodecReader> subs = new ArrayList<>();
- for (LeafReaderContext context : r.leaves()) {
- subs.add((CodecReader) context.reader());
- }
- if (VERBOSE) {
- System.out.println("TEST: now use addIndexes(CodecReader[]) to switch writers");
- }
- saveW.addIndexes(subs.toArray(new CodecReader[subs.size()]));
- } else {
- if (VERBOSE) {
- System.out.println("TEST: now use TestUtil.addIndexesSlowly(DirectoryReader[]) to switch writers");
- }
- TestUtil.addIndexesSlowly(saveW.w, r);
- }
- }
- } else {
- // Add via directory:
- if (VERBOSE) {
- System.out.println("TEST: now use addIndexes(Directory[]) to switch writers");
- }
- w.close();
- saveW.addIndexes(new Directory[] {dir});
- }
- w.close();
- dir.close();
- }
-
- private BigInteger randomBigInt(int numBytes) {
- BigInteger x = new BigInteger(numBytes*8-1, random());
- if (random().nextBoolean()) {
- x = x.negate();
- }
- return x;
- }
-
- private Directory getDirectory(int numPoints) throws IOException {
- Directory dir;
- if (numPoints > 100000) {
- dir = newFSDirectory(createTempDir("TestBKDTree"));
- } else {
- dir = newDirectory();
- }
- //dir = FSDirectory.open(createTempDir());
- return dir;
- }
-
- @Override
- protected boolean mergeIsStable() {
- // suppress this test from base class: merges for BKD trees are not stable because the tree created by merge will have a different
- // structure than the tree created by adding points separately
- return false;
- }
-}