You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/03/03 12:37:46 UTC
lucene-solr git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 97db2d0b9 -> 3c64325e9
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3c64325e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3c64325e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3c64325e
Branch: refs/heads/branch_6x
Commit: 3c64325e9362b454d4cdd4c8474ce143a9b13e99
Parents: 97db2d0
Author: Mike McCandless <mi...@apache.org>
Authored: Thu Mar 3 05:48:45 2016 -0500
Committer: Mike McCandless <mi...@apache.org>
Committed: Thu Mar 3 06:27:17 2016 -0500
----------------------------------------------------------------------
.../codecs/simpletext/SimpleTextBKDReader.java | 4 +-
.../simpletext/SimpleTextPointReader.java | 34 ++++++-
.../simpletext/SimpleTextPointWriter.java | 23 +++--
.../org/apache/lucene/codecs/PointFormat.java | 10 ++
.../org/apache/lucene/codecs/PointWriter.java | 10 ++
.../codecs/lucene60/Lucene60PointReader.java | 22 +++++
.../codecs/lucene60/Lucene60PointWriter.java | 6 +-
.../org/apache/lucene/index/CheckIndex.java | 96 ++++++++++++++++----
.../apache/lucene/index/ParallelLeafReader.java | 26 ++++++
.../org/apache/lucene/index/PointValues.java | 10 +-
.../apache/lucene/index/PointValuesWriter.java | 10 ++
.../lucene/index/SlowCodecReaderWrapper.java | 10 ++
.../org/apache/lucene/util/bkd/BKDReader.java | 23 ++++-
.../org/apache/lucene/util/bkd/BKDWriter.java | 31 ++++---
.../apache/lucene/index/TestPointValues.java | 23 +++++
.../org/apache/lucene/util/bkd/TestBKD.java | 15 ++-
.../apache/lucene/index/SortingLeafReader.java | 10 ++
.../codecs/asserting/AssertingPointFormat.java | 36 +++++++-
.../lucene/codecs/cranky/CrankyPointFormat.java | 10 ++
19 files changed, 350 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDReader.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDReader.java
index d0ab81e..09c40ec 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDReader.java
@@ -34,8 +34,8 @@ import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.BLOCK_VA
class SimpleTextBKDReader extends BKDReader {
public SimpleTextBKDReader(IndexInput datIn, int numDims, int maxPointsInLeafNode, int bytesPerDim, long[] leafBlockFPs, byte[] splitPackedValues,
- byte[] minPackedValue, byte[] maxPackedValue) throws IOException {
- super(datIn, numDims, maxPointsInLeafNode, bytesPerDim, leafBlockFPs, splitPackedValues, minPackedValue, maxPackedValue);
+ byte[] minPackedValue, byte[] maxPackedValue, long pointCount, int docCount) throws IOException {
+ super(datIn, numDims, maxPointsInLeafNode, bytesPerDim, leafBlockFPs, splitPackedValues, minPackedValue, maxPackedValue, pointCount, docCount);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointReader.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointReader.java
index 0ec2e03..05afd93 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointReader.java
@@ -39,6 +39,7 @@ import org.apache.lucene.util.bkd.BKDReader;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.BLOCK_FP;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.BYTES_PER_DIM;
+import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.DOC_COUNT;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.FIELD_COUNT;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.FIELD_FP;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.FIELD_FP_NAME;
@@ -47,6 +48,7 @@ import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.MAX_LEAF
import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.MAX_VALUE;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.MIN_VALUE;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.NUM_DIMS;
+import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.POINT_COUNT;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.SPLIT_COUNT;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.SPLIT_DIM;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointWriter.SPLIT_VALUE;
@@ -119,6 +121,14 @@ class SimpleTextPointReader extends PointReader {
assert startsWith(MAX_VALUE);
BytesRef maxValue = SimpleTextUtil.fromBytesRefString(stripPrefix(MAX_VALUE));
assert maxValue.length == numDims*bytesPerDim;
+
+ readLine(dataIn);
+ assert startsWith(POINT_COUNT);
+ long pointCount = parseLong(POINT_COUNT);
+
+ readLine(dataIn);
+ assert startsWith(DOC_COUNT);
+ int docCount = parseInt(DOC_COUNT);
long[] leafBlockFPs = new long[count];
for(int i=0;i<count;i++) {
@@ -139,7 +149,7 @@ class SimpleTextPointReader extends PointReader {
System.arraycopy(br.bytes, br.offset, splitPackedValues, (1 + bytesPerDim) * i + 1, bytesPerDim);
}
- return new SimpleTextBKDReader(dataIn, numDims, maxPointsInLeafNode, bytesPerDim, leafBlockFPs, splitPackedValues, minValue.bytes, maxValue.bytes);
+ return new SimpleTextBKDReader(dataIn, numDims, maxPointsInLeafNode, bytesPerDim, leafBlockFPs, splitPackedValues, minValue.bytes, maxValue.bytes, pointCount, docCount);
}
private void readLine(IndexInput in) throws IOException {
@@ -267,4 +277,26 @@ class SimpleTextPointReader extends PointReader {
}
return bkdReader.getBytesPerDimension();
}
+
+ @Override
+ public long size(String fieldName) {
+ BKDReader bkdReader = getBKDReader(fieldName);
+ if (bkdReader == null) {
+ // Schema ghost corner case! This field did index points in the past, but
+ // now all docs having this field were deleted in this segment:
+ return 0;
+ }
+ return bkdReader.getPointCount();
+ }
+
+ @Override
+ public int getDocCount(String fieldName) {
+ BKDReader bkdReader = getBKDReader(fieldName);
+ if (bkdReader == null) {
+ // Schema ghost corner case! This field did index points in the past, but
+ // now all docs having this field were deleted in this segment:
+ return 0;
+ }
+ return bkdReader.getDocCount();
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointWriter.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointWriter.java
index 0c5a3fc..a20e487 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointWriter.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointWriter.java
@@ -51,6 +51,8 @@ class SimpleTextPointWriter extends PointWriter {
final static BytesRef FIELD_FP = new BytesRef(" field fp ");
final static BytesRef MIN_VALUE = new BytesRef("min value ");
final static BytesRef MAX_VALUE = new BytesRef("max value ");
+ final static BytesRef POINT_COUNT = new BytesRef("point count ");
+ final static BytesRef DOC_COUNT = new BytesRef("doc count ");
private IndexOutput dataOut;
final BytesRefBuilder scratch = new BytesRefBuilder();
@@ -67,12 +69,13 @@ class SimpleTextPointWriter extends PointWriter {
public void writeField(FieldInfo fieldInfo, PointReader values) throws IOException {
// We use the normal BKDWriter, but subclass to customize how it writes the index and blocks to disk:
- try (BKDWriter writer = new BKDWriter(writeState.directory,
- writeState.segmentInfo.name,
- fieldInfo.getPointDimensionCount(),
- fieldInfo.getPointNumBytes(),
- BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE,
- BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP) {
+ try (BKDWriter writer = new BKDWriter(writeState.segmentInfo.maxDoc(),
+ writeState.directory,
+ writeState.segmentInfo.name,
+ fieldInfo.getPointDimensionCount(),
+ fieldInfo.getPointNumBytes(),
+ BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE,
+ BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP) {
@Override
protected void writeIndex(IndexOutput out, long[] leafBlockFPs, byte[] splitPackedValues) throws IOException {
@@ -102,6 +105,14 @@ class SimpleTextPointWriter extends PointWriter {
write(out, br.toString());
newline(out);
+ write(out, POINT_COUNT);
+ writeLong(out, pointCount);
+ newline(out);
+
+ write(out, DOC_COUNT);
+ writeInt(out, docsSeen.cardinality());
+ newline(out);
+
for(int i=0;i<leafBlockFPs.length;i++) {
write(out, BLOCK_FP);
writeLong(out, leafBlockFPs[i]);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/core/src/java/org/apache/lucene/codecs/PointFormat.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/PointFormat.java
index 1d48358..964f8f0 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/PointFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/PointFormat.java
@@ -95,6 +95,16 @@ public abstract class PointFormat {
public int getBytesPerDimension(String fieldName) {
throw new IllegalArgumentException("field=\"" + fieldName + "\" was not indexed with points");
}
+
+ @Override
+ public long size(String fieldName) {
+ throw new IllegalArgumentException("field=\"" + fieldName + "\" was not indexed with points");
+ }
+
+ @Override
+ public int getDocCount(String fieldName) {
+ throw new IllegalArgumentException("field=\"" + fieldName + "\" was not indexed with points");
+ }
};
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/core/src/java/org/apache/lucene/codecs/PointWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/PointWriter.java
index 21d9044..d244093 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/PointWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/PointWriter.java
@@ -115,6 +115,16 @@ public abstract class PointWriter implements Closeable {
public int getBytesPerDimension(String fieldName) {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public long size(String fieldName) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getDocCount(String fieldName) {
+ throw new UnsupportedOperationException();
+ }
});
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointReader.java
index 2e9206a..91a1e6c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointReader.java
@@ -215,5 +215,27 @@ public class Lucene60PointReader extends PointReader implements Closeable {
}
return bkdReader.getBytesPerDimension();
}
+
+ @Override
+ public long size(String fieldName) {
+ BKDReader bkdReader = getBKDReader(fieldName);
+ if (bkdReader == null) {
+ // Schema ghost corner case! This field did index points in the past, but
+ // now all docs having this point field were deleted in this segment:
+ return 0;
+ }
+ return bkdReader.getPointCount();
+ }
+
+ @Override
+ public int getDocCount(String fieldName) {
+ BKDReader bkdReader = getBKDReader(fieldName);
+ if (bkdReader == null) {
+ // Schema ghost corner case! This field did index points in the past, but
+ // now all docs having this point field were deleted in this segment:
+ return 0;
+ }
+ return bkdReader.getDocCount();
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java
index 9709ffb..63fc134 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java
@@ -82,7 +82,8 @@ public class Lucene60PointWriter extends PointWriter implements Closeable {
@Override
public void writeField(FieldInfo fieldInfo, PointReader values) throws IOException {
- try (BKDWriter writer = new BKDWriter(writeState.directory,
+ try (BKDWriter writer = new BKDWriter(writeState.segmentInfo.maxDoc(),
+ writeState.directory,
writeState.segmentInfo.name,
fieldInfo.getPointDimensionCount(),
fieldInfo.getPointNumBytes(),
@@ -129,7 +130,8 @@ public class Lucene60PointWriter extends PointWriter implements Closeable {
// Optimize the 1D case to use BKDWriter.merge, which does a single merge sort of the
// already sorted incoming segments, instead of trying to sort all points again as if
// we were simply reindexing them:
- try (BKDWriter writer = new BKDWriter(writeState.directory,
+ try (BKDWriter writer = new BKDWriter(writeState.segmentInfo.maxDoc(),
+ writeState.directory,
writeState.segmentInfo.name,
fieldInfo.getPointDimensionCount(),
fieldInfo.getPointNumBytes(),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
index 9f711df..1da69c1 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
@@ -1695,18 +1695,47 @@ public final class CheckIndex implements Closeable {
}
for (FieldInfo fieldInfo : fieldInfos) {
if (fieldInfo.getPointDimensionCount() > 0) {
+ FixedBitSet docsSeen = new FixedBitSet(reader.maxDoc());
status.totalValueFields++;
int dimCount = fieldInfo.getPointDimensionCount();
int bytesPerDim = fieldInfo.getPointNumBytes();
- byte[] lastMinPackedValue = new byte[dimCount*bytesPerDim];
- BytesRef lastMinPacked = new BytesRef(lastMinPackedValue);
- byte[] lastMaxPackedValue = new byte[dimCount*bytesPerDim];
- BytesRef lastMaxPacked = new BytesRef(lastMaxPackedValue);
+ int packedBytesCount = dimCount * bytesPerDim;
+ byte[] lastMinPackedValue = new byte[packedBytesCount];
+ byte[] lastMaxPackedValue = new byte[packedBytesCount];
BytesRef scratch = new BytesRef();
- lastMaxPacked.length = bytesPerDim;
- lastMinPacked.length = bytesPerDim;
scratch.length = bytesPerDim;
- byte[] lastPackedValue = new byte[dimCount*bytesPerDim];
+ byte[] lastPackedValue = new byte[packedBytesCount];
+
+ long[] pointCountSeen = new long[1];
+
+ byte[] globalMinPackedValue = values.getMinPackedValue(fieldInfo.name);
+ long size = values.size(fieldInfo.name);
+ int docCount = values.getDocCount(fieldInfo.name);
+
+ if (docCount > size) {
+ throw new RuntimeException("point values for field \"" + fieldInfo.name + "\" claims to have size=" + size + " points and inconsistent docCount=" + docCount);
+ }
+
+ if (docCount > reader.maxDoc()) {
+ throw new RuntimeException("point values for field \"" + fieldInfo.name + "\" claims to have docCount=" + docCount + " but that's greater than maxDoc=" + reader.maxDoc());
+ }
+
+ if (globalMinPackedValue == null) {
+ if (size != 0) {
+ throw new RuntimeException("getMinPackedValue is null points for field \"" + fieldInfo.name + "\" yet size=" + size);
+ }
+ } else if (globalMinPackedValue.length != packedBytesCount) {
+ throw new RuntimeException("getMinPackedValue for field \"" + fieldInfo.name + "\" return length=" + globalMinPackedValue.length + " array, but should be " + packedBytesCount);
+ }
+ byte[] globalMaxPackedValue = values.getMaxPackedValue(fieldInfo.name);
+ if (globalMaxPackedValue == null) {
+ if (size != 0) {
+ throw new RuntimeException("getMaxPackedValue is null points for field \"" + fieldInfo.name + "\" yet size=" + size);
+ }
+ } else if (globalMaxPackedValue.length != packedBytesCount) {
+ throw new RuntimeException("getMaxPackedValue for field \"" + fieldInfo.name + "\" return length=" + globalMaxPackedValue.length + " array, but should be " + packedBytesCount);
+ }
+
values.intersect(fieldInfo.name,
new PointValues.IntersectVisitor() {
@@ -1720,18 +1749,19 @@ public final class CheckIndex implements Closeable {
@Override
public void visit(int docID, byte[] packedValue) {
checkPackedValue("packed value", packedValue, docID);
- scratch.bytes = packedValue;
+ pointCountSeen[0]++;
+ docsSeen.set(docID);
for(int dim=0;dim<dimCount;dim++) {
- lastMaxPacked.offset = bytesPerDim * dim;
- lastMinPacked.offset = bytesPerDim * dim;
- scratch.offset = bytesPerDim * dim;
+ int offset = bytesPerDim * dim;
- if (scratch.compareTo(lastMinPacked) < 0) {
+ // Compare to last cell:
+ if (StringHelper.compare(bytesPerDim, packedValue, offset, lastMinPackedValue, offset) < 0) {
// This doc's point, in this dimension, is lower than the minimum value of the last cell checked:
throw new RuntimeException("packed points value " + Arrays.toString(packedValue) + " for docID=" + docID + " is out-of-bounds of the last cell min=" + Arrays.toString(lastMinPackedValue) + " max=" + Arrays.toString(lastMaxPackedValue) + " dim=" + dim);
}
- if (scratch.compareTo(lastMaxPacked) > 0) {
+
+ if (StringHelper.compare(bytesPerDim, packedValue, offset, lastMaxPackedValue, offset) > 0) {
// This doc's point, in this dimension, is greater than the maximum value of the last cell checked:
throw new RuntimeException("packed points value " + Arrays.toString(packedValue) + " for docID=" + docID + " is out-of-bounds of the last cell min=" + Arrays.toString(lastMinPackedValue) + " max=" + Arrays.toString(lastMaxPackedValue) + " dim=" + dim);
}
@@ -1758,9 +1788,33 @@ public final class CheckIndex implements Closeable {
@Override
public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
checkPackedValue("min packed value", minPackedValue, -1);
- System.arraycopy(minPackedValue, 0, lastMinPackedValue, 0, minPackedValue.length);
+ System.arraycopy(minPackedValue, 0, lastMinPackedValue, 0, packedBytesCount);
checkPackedValue("max packed value", maxPackedValue, -1);
- System.arraycopy(maxPackedValue, 0, lastMaxPackedValue, 0, maxPackedValue.length);
+ System.arraycopy(maxPackedValue, 0, lastMaxPackedValue, 0, packedBytesCount);
+
+ for(int dim=0;dim<dimCount;dim++) {
+ int offset = bytesPerDim * dim;
+
+ // Make sure this cell is not outside of the global min/max:
+ if (StringHelper.compare(bytesPerDim, minPackedValue, offset, globalMinPackedValue, offset) < 0) {
+ throw new RuntimeException("packed points cell minPackedValue " + Arrays.toString(minPackedValue) +
+ " is out-of-bounds of the global minimum " + Arrays.toString(globalMinPackedValue) + " dim=" + dim);
+ }
+
+ if (StringHelper.compare(bytesPerDim, maxPackedValue, offset, globalMinPackedValue, offset) < 0) {
+ throw new RuntimeException("packed points cell maxPackedValue " + Arrays.toString(minPackedValue) +
+ " is out-of-bounds of the global minimum " + Arrays.toString(globalMinPackedValue) + " dim=" + dim);
+ }
+
+ if (StringHelper.compare(bytesPerDim, minPackedValue, offset, globalMaxPackedValue, offset) > 0) {
+ throw new RuntimeException("packed points cell minPackedValue " + Arrays.toString(minPackedValue) +
+ " is out-of-bounds of the global maximum " + Arrays.toString(globalMaxPackedValue) + " dim=" + dim);
+ }
+ if (StringHelper.compare(bytesPerDim, maxPackedValue, offset, globalMaxPackedValue, offset) > 0) {
+ throw new RuntimeException("packed points cell maxPackedValue " + Arrays.toString(maxPackedValue) +
+ " is out-of-bounds of the global maximum " + Arrays.toString(globalMaxPackedValue) + " dim=" + dim);
+ }
+ }
// We always pretend the query shape is so complex that it crosses every cell, so
// that packedValue is passed for every document
@@ -1772,11 +1826,19 @@ public final class CheckIndex implements Closeable {
throw new RuntimeException(desc + " is null for docID=" + docID);
}
- if (packedValue.length != dimCount * bytesPerDim) {
- throw new RuntimeException(desc + " has incorrect length=" + packedValue.length + " vs expected=" + (dimCount * bytesPerDim) + " for docID=" + docID);
+ if (packedValue.length != packedBytesCount) {
+ throw new RuntimeException(desc + " has incorrect length=" + packedValue.length + " vs expected=" + packedBytesCount + " for docID=" + docID);
}
}
});
+
+ if (pointCountSeen[0] != size) {
+ throw new RuntimeException("point values for field \"" + fieldInfo.name + "\" claims to have size=" + size + " points, but in fact has " + pointCountSeen[0]);
+ }
+
+ if (docsSeen.cardinality() != docCount) {
+ throw new RuntimeException("point values for field \"" + fieldInfo.name + "\" claims to have docCount=" + docCount + " but in fact has " + docsSeen.cardinality());
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
index 3d51a0b..532265f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
@@ -381,6 +381,32 @@ public class ParallelLeafReader extends LeafReader {
}
return dimValues.getBytesPerDimension(fieldName);
}
+
+ @Override
+ public long size(String fieldName) {
+ LeafReader reader = fieldToReader.get(fieldName);
+ if (reader == null) {
+ return 0;
+ }
+ PointValues dimValues = reader.getPointValues();
+ if (dimValues == null) {
+ return 0;
+ }
+ return dimValues.size(fieldName);
+ }
+
+ @Override
+ public int getDocCount(String fieldName) {
+ LeafReader reader = fieldToReader.get(fieldName);
+ if (reader == null) {
+ return 0;
+ }
+ PointValues dimValues = reader.getPointValues();
+ if (dimValues == null) {
+ return 0;
+ }
+ return dimValues.getDocCount(fieldName);
+ }
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/core/src/java/org/apache/lucene/index/PointValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValues.java b/lucene/core/src/java/org/apache/lucene/index/PointValues.java
index 34d315f..230a14f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PointValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PointValues.java
@@ -80,10 +80,10 @@ public abstract class PointValues {
* to test whether each document is deleted, if necessary. */
public abstract void intersect(String fieldName, IntersectVisitor visitor) throws IOException;
- /** Returns minimum value for each dimension, packed, or null if no points were indexed */
+ /** Returns minimum value for each dimension, packed, or null if {@link #size} is <code>0</code> */
public abstract byte[] getMinPackedValue(String fieldName) throws IOException;
- /** Returns maximum value for each dimension, packed, or null if no points were indexed */
+ /** Returns maximum value for each dimension, packed, or null if {@link #size} is <code>0</code> */
public abstract byte[] getMaxPackedValue(String fieldName) throws IOException;
/** Returns how many dimensions were indexed */
@@ -91,4 +91,10 @@ public abstract class PointValues {
/** Returns the number of bytes per dimension */
public abstract int getBytesPerDimension(String fieldName) throws IOException;
+
+ /** Returns the total number of indexed points across all documents in this field. */
+ public abstract long size(String fieldName);
+
+ /** Returns the total number of documents that have indexed at least one point for this field. */
+ public abstract int getDocCount(String fieldName);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
index 2fa8b4f..546bf71 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
@@ -108,6 +108,16 @@ class PointValuesWriter {
public int getBytesPerDimension(String fieldName) {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public long size(String fieldName) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getDocCount(String fieldName) {
+ throw new UnsupportedOperationException();
+ }
});
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
index 50d9778..a741111 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
@@ -172,6 +172,16 @@ public final class SlowCodecReaderWrapper {
public int getBytesPerDimension(String fieldName) throws IOException {
return values.getBytesPerDimension(fieldName);
}
+
+ @Override
+ public long size(String fieldName) {
+ return values.size(fieldName);
+ }
+
+ @Override
+ public int getDocCount(String fieldName) {
+ return values.getDocCount(fieldName);
+ }
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
index 9d72599..b904a50 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
@@ -42,6 +42,8 @@ public class BKDReader implements Accountable {
final int maxPointsInLeafNode;
final byte[] minPackedValue;
final byte[] maxPackedValue;
+ final long pointCount;
+ final int docCount;
protected final int packedBytesLength;
/** Caller must pre-seek the provided {@link IndexInput} to the index location that {@link BKDWriter#finish} returned */
@@ -59,9 +61,13 @@ public class BKDReader implements Accountable {
minPackedValue = new byte[packedBytesLength];
maxPackedValue = new byte[packedBytesLength];
+
in.readBytes(minPackedValue, 0, packedBytesLength);
in.readBytes(maxPackedValue, 0, packedBytesLength);
+ pointCount = in.readVLong();
+ docCount = in.readVInt();
+
splitPackedValues = new byte[(1+bytesPerDim)*numLeaves];
// TODO: don't write split packed values[0]!
@@ -122,7 +128,7 @@ public class BKDReader implements Accountable {
/** Called by consumers that have their own on-disk format for the index (e.g. SimpleText) */
protected BKDReader(IndexInput in, int numDims, int maxPointsInLeafNode, int bytesPerDim, long[] leafBlockFPs, byte[] splitPackedValues,
- byte[] minPackedValue, byte[] maxPackedValue) throws IOException {
+ byte[] minPackedValue, byte[] maxPackedValue, long pointCount, int docCount) throws IOException {
this.in = in;
this.numDims = numDims;
this.maxPointsInLeafNode = maxPointsInLeafNode;
@@ -133,6 +139,8 @@ public class BKDReader implements Accountable {
this.splitPackedValues = splitPackedValues;
this.minPackedValue = minPackedValue;
this.maxPackedValue = maxPackedValue;
+ this.pointCount = pointCount;
+ this.docCount = docCount;
assert minPackedValue.length == packedBytesLength;
assert maxPackedValue.length == packedBytesLength;
}
@@ -275,10 +283,7 @@ public class BKDReader implements Accountable {
packedBytesLength,
maxPointsInLeafNode,
visitor);
- byte[] rootMinPacked = new byte[packedBytesLength];
- byte[] rootMaxPacked = new byte[packedBytesLength];
- Arrays.fill(rootMaxPacked, (byte) 0xff);
- intersect(state, 1, rootMinPacked, rootMaxPacked);
+ intersect(state, 1, minPackedValue, maxPackedValue);
}
/** Fast path: this is called when the query box fully encompasses all cells under this node. */
@@ -430,4 +435,12 @@ public class BKDReader implements Accountable {
public int getBytesPerDimension() {
return bytesPerDim;
}
+
+ public long getPointCount() {
+ return pointCount;
+ }
+
+ public int getDocCount() {
+ return docCount;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 9d6ad97..5388bb3 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -34,6 +34,7 @@ import org.apache.lucene.store.TrackingDirectoryWrapper;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntroSorter;
import org.apache.lucene.util.LongBitSet;
@@ -110,6 +111,8 @@ public class BKDWriter implements Closeable {
final byte[] scratch2;
final int[] commonPrefixLengths;
+ protected final FixedBitSet docsSeen;
+
private OfflinePointWriter offlinePointWriter;
private HeapPointWriter heapPointWriter;
@@ -123,13 +126,13 @@ public class BKDWriter implements Closeable {
/** Maximum per-dim values, packed */
protected final byte[] maxPackedValue;
- private long pointCount;
+ protected long pointCount;
- public BKDWriter(Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim) throws IOException {
- this(tempDir, tempFileNamePrefix, numDims, bytesPerDim, DEFAULT_MAX_POINTS_IN_LEAF_NODE, DEFAULT_MAX_MB_SORT_IN_HEAP);
+ public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim) throws IOException {
+ this(maxDoc, tempDir, tempFileNamePrefix, numDims, bytesPerDim, DEFAULT_MAX_POINTS_IN_LEAF_NODE, DEFAULT_MAX_MB_SORT_IN_HEAP);
}
- public BKDWriter(Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim, int maxPointsInLeafNode, double maxMBSortInHeap) throws IOException {
+ public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim, int maxPointsInLeafNode, double maxMBSortInHeap) throws IOException {
verifyParams(numDims, maxPointsInLeafNode, maxMBSortInHeap);
// We use tracking dir to deal with removing files on exception, so each place that
// creates temp files doesn't need crazy try/finally/sucess logic:
@@ -138,6 +141,7 @@ public class BKDWriter implements Closeable {
this.maxPointsInLeafNode = maxPointsInLeafNode;
this.numDims = numDims;
this.bytesPerDim = bytesPerDim;
+ docsSeen = new FixedBitSet(maxDoc);
packedBytesLength = numDims * bytesPerDim;
scratchDiff = new byte[bytesPerDim];
@@ -239,6 +243,7 @@ public class BKDWriter implements Closeable {
}
pointCount++;
+ docsSeen.set(docID);
}
/** How many points have been added so far */
@@ -420,15 +425,18 @@ public class BKDWriter implements Closeable {
// System.out.println("iter reader=" + reader);
// NOTE: doesn't work with subclasses (e.g. SimpleText!)
- leafBlockDocIDs[leafCount] = reader.docIDBase + reader.docID;
+ int docID = reader.docIDBase + reader.docID;
+ leafBlockDocIDs[leafCount] = docID;
System.arraycopy(reader.state.scratchPackedValue, 0, leafBlockPackedValues[leafCount], 0, packedBytesLength);
+ docsSeen.set(docID);
if (valueCount == 0) {
System.arraycopy(reader.state.scratchPackedValue, 0, minPackedValue, 0, packedBytesLength);
}
System.arraycopy(reader.state.scratchPackedValue, 0, maxPackedValue, 0, packedBytesLength);
- assert numDims > 1 || valueInOrder(valueCount++, lastPackedValue, reader.state.scratchPackedValue);
+ assert numDims > 1 || valueInOrder(valueCount, lastPackedValue, reader.state.scratchPackedValue);
+ valueCount++;
if (leafCount == 0) {
if (leafBlockFPs.size() > 0) {
@@ -478,6 +486,8 @@ public class BKDWriter implements Closeable {
}
}
+ pointCount = valueCount;
+
long indexFP = out.getFilePointer();
int numInnerNodes = leafBlockStartValues.size();
@@ -799,10 +809,6 @@ public class BKDWriter implements Closeable {
// Sort all docs once by each dimension:
PathSlice[] sortedPointWriters = new PathSlice[numDims];
- byte[] minPacked = new byte[packedBytesLength];
- byte[] maxPacked = new byte[packedBytesLength];
- Arrays.fill(maxPacked, (byte) 0xff);
-
boolean success = false;
try {
//long t0 = System.nanoTime();
@@ -822,7 +828,7 @@ public class BKDWriter implements Closeable {
build(1, numLeaves, sortedPointWriters,
ordBitSet, out,
- minPacked, maxPacked,
+ minPackedValue, maxPackedValue,
splitPackedValues,
leafBlockFPs);
@@ -862,6 +868,9 @@ public class BKDWriter implements Closeable {
out.writeBytes(minPackedValue, 0, packedBytesLength);
out.writeBytes(maxPackedValue, 0, packedBytesLength);
+ out.writeVLong(pointCount);
+ out.writeVInt(docsSeen.cardinality());
+
// TODO: for 1D case, don't waste the first byte of each split value (it's always 0)
// NOTE: splitPackedValues[0] is unused, because nodeID is 1-based:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
index 7551d3c..9aafb3f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
@@ -34,6 +34,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
import org.apache.lucene.index.PointValues;
@@ -540,4 +541,26 @@ public class TestPointValues extends LuceneTestCase {
w.close();
dir.close();
}
+
+ public void testDeleteAllPointDocs() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig();
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.NO));
+ doc.add(new IntPoint("int", 17));
+ w.addDocument(doc);
+ w.addDocument(new Document());
+ w.commit();
+
+ w.deleteDocuments(new Term("id", "0"));
+
+ w.forceMerge(1);
+ DirectoryReader r = w.getReader();
+ assertEquals(0, r.leaves().get(0).reader().getPointValues().size("int"));
+ assertEquals(0, r.leaves().get(0).reader().getPointValues().getDocCount("int"));
+ w.close();
+ r.close();
+ dir.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
index 1688e6f..2be2b42 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
@@ -42,7 +42,7 @@ public class TestBKD extends LuceneTestCase {
public void testBasicInts1D() throws Exception {
try (Directory dir = getDirectory(100)) {
- BKDWriter w = new BKDWriter(dir, "tmp", 1, 4, 2, 1.0f);
+ BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 4, 2, 1.0f);
byte[] scratch = new byte[4];
for(int docID=0;docID<100;docID++) {
NumericUtils.intToBytes(docID, scratch, 0);
@@ -117,7 +117,7 @@ public class TestBKD extends LuceneTestCase {
int numDims = TestUtil.nextInt(random(), 1, 5);
int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100);
float maxMB = (float) 3.0 + (3*random().nextFloat());
- BKDWriter w = new BKDWriter(dir, "tmp", numDims, 4, maxPointsInLeafNode, maxMB);
+ BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, 4, maxPointsInLeafNode, maxMB);
if (VERBOSE) {
System.out.println("TEST: numDims=" + numDims + " numDocs=" + numDocs);
@@ -258,7 +258,7 @@ public class TestBKD extends LuceneTestCase {
int numDims = TestUtil.nextInt(random(), 1, 5);
int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100);
float maxMB = (float) 3.0 + (3*random().nextFloat());
- BKDWriter w = new BKDWriter(dir, "tmp", numDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
+ BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
BigInteger[][] docs = new BigInteger[numDocs][];
byte[] scratch = new byte[numBytesPerDim*numDims];
@@ -431,7 +431,7 @@ public class TestBKD extends LuceneTestCase {
public void testTooLittleHeap() throws Exception {
try (Directory dir = getDirectory(0)) {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
- new BKDWriter(dir, "bkd", 1, 16, 1000000, 0.001);
+ new BKDWriter(1, dir, "bkd", 1, 16, 1000000, 0.001);
});
assertTrue(expected.getMessage().contains("either increase maxMBSortInHeap or decrease maxPointsInLeafNode"));
}
@@ -631,7 +631,7 @@ public class TestBKD extends LuceneTestCase {
List<Integer> docIDBases = null;
int seg = 0;
- BKDWriter w = new BKDWriter(dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
+ BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT);
IndexInput in = null;
@@ -685,7 +685,7 @@ public class TestBKD extends LuceneTestCase {
seg++;
maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 1000);
maxMB = (float) 3.0 + (3*random().nextDouble());
- w = new BKDWriter(dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
+ w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
lastDocIDBase = docID;
}
}
@@ -693,7 +693,6 @@ public class TestBKD extends LuceneTestCase {
long indexFP;
if (toMerge != null) {
- System.out.println("merge " + toMerge.size());
if (segCount > 0) {
docIDBases.add(lastDocIDBase);
toMerge.add(w.finish(out));
@@ -701,7 +700,7 @@ public class TestBKD extends LuceneTestCase {
out.close();
in = dir.openInput("bkd", IOContext.DEFAULT);
seg++;
- w = new BKDWriter(dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
+ w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
List<BKDReader> readers = new ArrayList<>();
for(long fp : toMerge) {
in.seek(fp);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/misc/src/java/org/apache/lucene/index/SortingLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/java/org/apache/lucene/index/SortingLeafReader.java b/lucene/misc/src/java/org/apache/lucene/index/SortingLeafReader.java
index 12cee20..683c5c2 100644
--- a/lucene/misc/src/java/org/apache/lucene/index/SortingLeafReader.java
+++ b/lucene/misc/src/java/org/apache/lucene/index/SortingLeafReader.java
@@ -303,6 +303,16 @@ public class SortingLeafReader extends FilterLeafReader {
public int getBytesPerDimension(String fieldName) throws IOException {
return in.getBytesPerDimension(fieldName);
}
+
+ @Override
+ public long size(String fieldName) {
+ return in.size(fieldName);
+ }
+
+ @Override
+ public int getDocCount(String fieldName) {
+ return in.getDocCount(fieldName);
+ }
}
private static class SortingSortedDocValues extends SortedDocValues {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java
index 15836de..07365e0 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java
@@ -31,6 +31,7 @@ import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.TestUtil;
@@ -63,7 +64,7 @@ public final class AssertingPointFormat extends PointFormat {
@Override
public PointReader fieldsReader(SegmentReadState state) throws IOException {
- return new AssertingPointReader(in.fieldsReader(state));
+ return new AssertingPointReader(state.segmentInfo.maxDoc(), in.fieldsReader(state));
}
/** Validates in the 1D case that all points are visited in order, and point values are in bounds of the last cell checked */
@@ -105,8 +106,8 @@ public final class AssertingPointFormat extends PointFormat {
// This doc's packed value should be contained in the last cell passed to compare:
for(int dim=0;dim<numDims;dim++) {
- assert StringHelper.compare(bytesPerDim, lastMinPackedValue, dim*bytesPerDim, packedValue, dim*bytesPerDim) <= 0: "dim=" + dim + " of " + numDims;
- assert StringHelper.compare(bytesPerDim, lastMaxPackedValue, dim*bytesPerDim, packedValue, dim*bytesPerDim) >= 0: "dim=" + dim + " of " + numDims;
+ assert StringHelper.compare(bytesPerDim, lastMinPackedValue, dim*bytesPerDim, packedValue, dim*bytesPerDim) <= 0: "dim=" + dim + " of " + numDims + " value=" + new BytesRef(packedValue);
+ assert StringHelper.compare(bytesPerDim, lastMaxPackedValue, dim*bytesPerDim, packedValue, dim*bytesPerDim) >= 0: "dim=" + dim + " of " + numDims + " value=" + new BytesRef(packedValue);
}
// TODO: we should assert that this "matches" whatever relation the last call to compare had returned
@@ -145,9 +146,11 @@ public final class AssertingPointFormat extends PointFormat {
static class AssertingPointReader extends PointReader {
private final PointReader in;
+ private final int maxDoc;
- AssertingPointReader(PointReader in) {
+ AssertingPointReader(int maxDoc, PointReader in) {
this.in = in;
+ this.maxDoc = maxDoc;
// do a few simple checks on init
assert toString() != null;
assert ramBytesUsed() >= 0;
@@ -187,7 +190,7 @@ public final class AssertingPointFormat extends PointFormat {
@Override
public PointReader getMergeInstance() throws IOException {
- return new AssertingPointReader(in.getMergeInstance());
+ return new AssertingPointReader(maxDoc, in.getMergeInstance());
}
@Override
@@ -197,23 +200,46 @@ public final class AssertingPointFormat extends PointFormat {
@Override
public byte[] getMinPackedValue(String fieldName) throws IOException {
+ assertStats(fieldName);
return in.getMinPackedValue(fieldName);
}
@Override
public byte[] getMaxPackedValue(String fieldName) throws IOException {
+ assertStats(fieldName);
return in.getMaxPackedValue(fieldName);
}
@Override
public int getNumDimensions(String fieldName) throws IOException {
+ assertStats(fieldName);
return in.getNumDimensions(fieldName);
}
@Override
public int getBytesPerDimension(String fieldName) throws IOException {
+ assertStats(fieldName);
return in.getBytesPerDimension(fieldName);
}
+
+ @Override
+ public long size(String fieldName) {
+ assertStats(fieldName);
+ return in.size(fieldName);
+ }
+
+ @Override
+ public int getDocCount(String fieldName) {
+ assertStats(fieldName);
+ return in.getDocCount(fieldName);
+ }
+
+ private void assertStats(String fieldName) {
+ assert in.size(fieldName) >= 0;
+ assert in.getDocCount(fieldName) >= 0;
+ assert in.getDocCount(fieldName) <= in.size(fieldName);
+ assert in.getDocCount(fieldName) <= maxDoc;
+ }
}
static class AssertingPointWriter extends PointWriter {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c64325e/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointFormat.java
index 6b83b1e..b663a80 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointFormat.java
@@ -171,5 +171,15 @@ class CrankyPointFormat extends PointFormat {
public long ramBytesUsed() {
return delegate.ramBytesUsed();
}
+
+ @Override
+ public long size(String fieldName) {
+ return delegate.size(fieldName);
+ }
+
+ @Override
+ public int getDocCount(String fieldName) {
+ return delegate.getDocCount(fieldName);
+ }
}
}