You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by nk...@apache.org on 2018/10/09 00:06:32 UTC
[1/4] lucene-solr:master: LUCENE-8496: Selective indexing - modify
BKDReader/BKDWriter to allow users to select a fewer number of dimensions to
be used for creating the index than the total number of dimensions used for
field encoding. i.e., dimensions 0
Repository: lucene-solr
Updated Branches:
refs/heads/master df07a43b5 -> 1118299c3
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
index 4bd806e..aa12de7 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
@@ -1048,7 +1048,7 @@ public class AssertingLeafReader extends FilterLeafReader {
@Override
public void intersect(IntersectVisitor visitor) throws IOException {
- in.intersect(new AssertingIntersectVisitor(in.getNumDimensions(), in.getBytesPerDimension(), visitor));
+ in.intersect(new AssertingIntersectVisitor(in.getNumDataDimensions(), in.getNumIndexDimensions(), in.getBytesPerDimension(), visitor));
}
@Override
@@ -1069,8 +1069,13 @@ public class AssertingLeafReader extends FilterLeafReader {
}
@Override
- public int getNumDimensions() throws IOException {
- return in.getNumDimensions();
+ public int getNumDataDimensions() throws IOException {
+ return in.getNumDataDimensions();
+ }
+
+ @Override
+ public int getNumIndexDimensions() throws IOException {
+ return in.getNumIndexDimensions();
}
@Override
@@ -1093,7 +1098,8 @@ public class AssertingLeafReader extends FilterLeafReader {
/** Validates in the 1D case that all points are visited in order, and point values are in bounds of the last cell checked */
static class AssertingIntersectVisitor implements IntersectVisitor {
final IntersectVisitor in;
- final int numDims;
+ final int numDataDims;
+ final int numIndexDims;
final int bytesPerDim;
final byte[] lastDocValue;
final byte[] lastMinPackedValue;
@@ -1102,13 +1108,14 @@ public class AssertingLeafReader extends FilterLeafReader {
private int lastDocID = -1;
private int docBudget;
- AssertingIntersectVisitor(int numDims, int bytesPerDim, IntersectVisitor in) {
+ AssertingIntersectVisitor(int numDataDims, int numIndexDims, int bytesPerDim, IntersectVisitor in) {
this.in = in;
- this.numDims = numDims;
+ this.numDataDims = numDataDims;
+ this.numIndexDims = numIndexDims;
this.bytesPerDim = bytesPerDim;
- lastMaxPackedValue = new byte[numDims*bytesPerDim];
- lastMinPackedValue = new byte[numDims*bytesPerDim];
- if (numDims == 1) {
+ lastMaxPackedValue = new byte[numDataDims*bytesPerDim];
+ lastMinPackedValue = new byte[numDataDims*bytesPerDim];
+ if (numDataDims == 1) {
lastDocValue = new byte[bytesPerDim];
} else {
lastDocValue = null;
@@ -1132,14 +1139,14 @@ public class AssertingLeafReader extends FilterLeafReader {
assert lastCompareResult == PointValues.Relation.CELL_CROSSES_QUERY;
// This doc's packed value should be contained in the last cell passed to compare:
- for(int dim=0;dim<numDims;dim++) {
- assert FutureArrays.compareUnsigned(lastMinPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim, packedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim) <= 0: "dim=" + dim + " of " + numDims + " value=" + new BytesRef(packedValue);
- assert FutureArrays.compareUnsigned(lastMaxPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim, packedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim) >= 0: "dim=" + dim + " of " + numDims + " value=" + new BytesRef(packedValue);
+ for(int dim=0;dim<numIndexDims;dim++) {
+ assert FutureArrays.compareUnsigned(lastMinPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim, packedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim) <= 0: "dim=" + dim + " of " + numDataDims + " value=" + new BytesRef(packedValue);
+ assert FutureArrays.compareUnsigned(lastMaxPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim, packedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim) >= 0: "dim=" + dim + " of " + numDataDims + " value=" + new BytesRef(packedValue);
}
// TODO: we should assert that this "matches" whatever relation the last call to compare had returned
- assert packedValue.length == numDims * bytesPerDim;
- if (numDims == 1) {
+ assert packedValue.length == numDataDims * bytesPerDim;
+ if (numDataDims == 1) {
int cmp = FutureArrays.compareUnsigned(lastDocValue, 0, bytesPerDim, packedValue, 0, bytesPerDim);
if (cmp < 0) {
// ok
@@ -1163,11 +1170,11 @@ public class AssertingLeafReader extends FilterLeafReader {
@Override
public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
assert FutureArrays.compareUnsigned(minPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim, maxPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim) <= 0;
}
- System.arraycopy(maxPackedValue, 0, lastMaxPackedValue, 0, numDims*bytesPerDim);
- System.arraycopy(minPackedValue, 0, lastMinPackedValue, 0, numDims*bytesPerDim);
+ System.arraycopy(maxPackedValue, 0, lastMaxPackedValue, 0, numIndexDims*bytesPerDim);
+ System.arraycopy(minPackedValue, 0, lastMinPackedValue, 0, numIndexDims*bytesPerDim);
lastCompareResult = in.compare(minPackedValue, maxPackedValue);
return lastCompareResult;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
index 03f01e7..05ed0ff 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
@@ -325,7 +325,7 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
FieldInfo proto = oneDocReader.getFieldInfos().fieldInfo("field");
FieldInfo field = new FieldInfo(proto.name, proto.number, proto.hasVectors(), proto.omitsNorms(), proto.hasPayloads(),
proto.getIndexOptions(), proto.getDocValuesType(), proto.getDocValuesGen(), new HashMap<>(),
- proto.getPointDimensionCount(), proto.getPointNumBytes(), proto.isSoftDeletesField());
+ proto.getPointDataDimensionCount(), proto.getPointIndexDimensionCount(), proto.getPointNumBytes(), proto.isSoftDeletesField());
FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { field } );
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java
index b2e3e4e..71a9f26 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java
@@ -28,6 +28,7 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.BinaryPoint;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StringField;
@@ -520,13 +521,14 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
private void doTestRandomBinary(int count) throws Exception {
int numDocs = TestUtil.nextInt(random(), count, count*2);
int numBytesPerDim = TestUtil.nextInt(random(), 2, PointValues.MAX_NUM_BYTES);
- int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS);
+ int numDataDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS);
+ int numIndexDims = TestUtil.nextInt(random(), 1, numDataDims);
byte[][][] docValues = new byte[numDocs][][];
for(int docID=0;docID<numDocs;docID++) {
- byte[][] values = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
+ byte[][] values = new byte[numDataDims][];
+ for(int dim=0;dim<numDataDims;dim++) {
values[dim] = new byte[numBytesPerDim];
// TODO: sometimes test on a "small" volume too, so we test the high density cases, higher chance of boundary, etc. cases:
random().nextBytes(values[dim]);
@@ -534,17 +536,22 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
docValues[docID] = values;
}
- verify(docValues, null, numDims, numBytesPerDim);
+ verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim);
}
- /** docIDs can be null, for the single valued case, else it maps value to docID, but all values for one doc must be adjacent */
private void verify(byte[][][] docValues, int[] docIDs, int numDims, int numBytesPerDim) throws Exception {
+ verify(docValues, docIDs, numDims, numDims, numBytesPerDim);
+ }
+
+ /** docIDs can be null, for the single valued case, else it maps value to docID, but all values for one doc must be adjacent */
+ private void verify(byte[][][] docValues, int[] docIDs, int numDataDims, int numIndexDims, int numBytesPerDim) throws Exception {
try (Directory dir = getDirectory(docValues.length)) {
while (true) {
try {
- verify(dir, docValues, docIDs, numDims, numBytesPerDim, false);
+ verify(dir, docValues, docIDs, numDataDims, numIndexDims, numBytesPerDim, false);
return;
} catch (IllegalArgumentException iae) {
+ iae.printStackTrace();
// This just means we got a too-small maxMB for the maxPointsInLeafNode; just retry
assertTrue(iae.getMessage().contains("either increase maxMBSortInHeap or decrease maxPointsInLeafNode"));
}
@@ -553,9 +560,22 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
}
private void verify(Directory dir, byte[][][] docValues, int[] ids, int numDims, int numBytesPerDim, boolean expectExceptions) throws Exception {
+ verify(dir, docValues, ids, numDims, numDims, numBytesPerDim, expectExceptions);
+ }
+
+ private byte[] flattenBinaryPoint(byte[][] value, int numDataDims, int numBytesPerDim) {
+ byte[] result = new byte[value.length * numBytesPerDim];
+ for (int d = 0; d < numDataDims; ++d) {
+ System.arraycopy(value[d], 0, result, d * numBytesPerDim, numBytesPerDim);
+ }
+ return result;
+ }
+
+ /** test selective indexing */
+ private void verify(Directory dir, byte[][][] docValues, int[] ids, int numDataDims, int numIndexDims, int numBytesPerDim, boolean expectExceptions) throws Exception {
int numValues = docValues.length;
if (VERBOSE) {
- System.out.println("TEST: numValues=" + numValues + " numDims=" + numDims + " numBytesPerDim=" + numBytesPerDim);
+ System.out.println("TEST: numValues=" + numValues + " numDataDims=" + numDataDims + " numIndexDims=" + numIndexDims + " numBytesPerDim=" + numBytesPerDim);
}
// RandomIndexWriter is too slow:
@@ -578,10 +598,10 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
DirectoryReader r = null;
// Compute actual min/max values:
- byte[][] expectedMinValues = new byte[numDims][];
- byte[][] expectedMaxValues = new byte[numDims][];
+ byte[][] expectedMinValues = new byte[numDataDims][];
+ byte[][] expectedMaxValues = new byte[numDataDims][];
for(int ord=0;ord<docValues.length;ord++) {
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
if (ord == 0) {
expectedMinValues[dim] = new byte[numBytesPerDim];
System.arraycopy(docValues[ord][dim], 0, expectedMinValues[dim], 0, numBytesPerDim);
@@ -629,6 +649,10 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
try {
+ FieldType fieldType = new FieldType();
+ fieldType.setDimensions(numDataDims, numIndexDims, numBytesPerDim);
+ fieldType.freeze();
+
Document doc = null;
int lastID = -1;
for(int ord=0;ord<numValues;ord++) {
@@ -649,7 +673,10 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
doc = new Document();
doc.add(new NumericDocValuesField("id", id));
}
- doc.add(new BinaryPoint("field", docValues[ord]));
+ // pack the binary point
+ byte[] val = flattenBinaryPoint(docValues[ord], numDataDims, numBytesPerDim);
+
+ doc.add(new BinaryPoint("field", val, fieldType));
lastID = id;
if (random().nextInt(30) == 17) {
@@ -667,7 +694,8 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
if (random().nextInt(30) == 17) {
// randomly index some documents with this field, but we will delete them:
Document xdoc = new Document();
- xdoc.add(new BinaryPoint("field", docValues[ord]));
+ val = flattenBinaryPoint(docValues[ord], numDataDims, numBytesPerDim);
+ xdoc.add(new BinaryPoint("field", val, fieldType));
xdoc.add(new StringField("nukeme", "yes", Field.Store.NO));
if (useRealWriter) {
w.w.addDocument(xdoc);
@@ -689,7 +717,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
if (VERBOSE) {
System.out.println(" ord=" + ord + " id=" + id);
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
System.out.println(" dim=" + dim + " value=" + new BytesRef(docValues[ord][dim]));
}
}
@@ -731,10 +759,10 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
Bits liveDocs = MultiFields.getLiveDocs(r);
// Verify min/max values are correct:
- byte[] minValues = new byte[numDims*numBytesPerDim];
+ byte[] minValues = new byte[numIndexDims*numBytesPerDim];
Arrays.fill(minValues, (byte) 0xff);
- byte[] maxValues = new byte[numDims*numBytesPerDim];
+ byte[] maxValues = new byte[numIndexDims*numBytesPerDim];
for(LeafReaderContext ctx : r.leaves()) {
PointValues dimValues = ctx.reader().getPointValues("field");
@@ -744,7 +772,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
byte[] leafMinValues = dimValues.getMinPackedValue();
byte[] leafMaxValues = dimValues.getMaxPackedValue();
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
if (FutureArrays.compareUnsigned(leafMinValues, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, minValues, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim) < 0) {
System.arraycopy(leafMinValues, dim*numBytesPerDim, minValues, dim*numBytesPerDim, numBytesPerDim);
}
@@ -755,7 +783,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
}
byte[] scratch = new byte[numBytesPerDim];
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
System.arraycopy(minValues, dim*numBytesPerDim, scratch, 0, numBytesPerDim);
//System.out.println("dim=" + dim + " expectedMin=" + new BytesRef(expectedMinValues[dim]) + " min=" + new BytesRef(scratch));
assertTrue(Arrays.equals(expectedMinValues[dim], scratch));
@@ -771,9 +799,9 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
}
// Random N dims rect query:
- byte[][] queryMin = new byte[numDims][];
- byte[][] queryMax = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
+ byte[][] queryMin = new byte[numIndexDims][];
+ byte[][] queryMax = new byte[numIndexDims][];
+ for(int dim=0;dim<numIndexDims;dim++) {
queryMin[dim] = new byte[numBytesPerDim];
random().nextBytes(queryMin[dim]);
queryMax[dim] = new byte[numBytesPerDim];
@@ -786,7 +814,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
}
if (VERBOSE) {
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
System.out.println(" dim=" + dim + "\n queryMin=" + new BytesRef(queryMin[dim]) + "\n queryMax=" + new BytesRef(queryMax[dim]));
}
}
@@ -816,7 +844,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
return;
}
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
//System.out.println(" dim=" + dim + " value=" + new BytesRef(packedValue, dim*numBytesPerDim, numBytesPerDim));
if (FutureArrays.compareUnsigned(packedValue, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMin[dim], 0, numBytesPerDim) < 0 ||
FutureArrays.compareUnsigned(packedValue, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMax[dim], 0, numBytesPerDim) > 0) {
@@ -833,7 +861,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
public Relation compare(byte[] minPacked, byte[] maxPacked) {
boolean crosses = false;
//System.out.println("compare");
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
if (FutureArrays.compareUnsigned(maxPacked, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMin[dim], 0, numBytesPerDim) < 0 ||
FutureArrays.compareUnsigned(minPacked, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMax[dim], 0, numBytesPerDim) > 0) {
//System.out.println(" query_outside_cell");
@@ -858,7 +886,7 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
BitSet expected = new BitSet();
for(int ord=0;ord<numValues;ord++) {
boolean matches = true;
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
byte[] x = docValues[ord][dim];
if (FutureArrays.compareUnsigned(x, 0, numBytesPerDim, queryMin[dim], 0, numBytesPerDim) < 0 ||
FutureArrays.compareUnsigned(x, 0, numBytesPerDim, queryMax[dim], 0, numBytesPerDim) > 0) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java
index 2c74677..691f19e 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/MismatchedLeafReader.java
@@ -76,7 +76,8 @@ public class MismatchedLeafReader extends FilterLeafReader {
oldInfo.getDocValuesType(), // docValuesType
oldInfo.getDocValuesGen(), // dvGen
oldInfo.attributes(), // attributes
- oldInfo.getPointDimensionCount(), // dimension count
+ oldInfo.getPointDataDimensionCount(), // data dimension count
+ oldInfo.getPointIndexDimensionCount(), // index dimension count
oldInfo.getPointNumBytes(), // dimension numBytes
oldInfo.isSoftDeletesField()); // used as soft-deletes field
shuffled.set(i, newInfo);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
index 3588a8e..b5a728b 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
@@ -111,7 +111,8 @@ public class RandomCodec extends AssertingCodec {
try (BKDWriter writer = new RandomlySplittingBKDWriter(writeState.segmentInfo.maxDoc(),
writeState.directory,
writeState.segmentInfo.name,
- fieldInfo.getPointDimensionCount(),
+ fieldInfo.getPointDataDimensionCount(),
+ fieldInfo.getPointIndexDimensionCount(),
fieldInfo.getPointNumBytes(),
maxPointsInLeafNode,
maxMBSortInHeap,
@@ -259,10 +260,10 @@ public class RandomCodec extends AssertingCodec {
final Random random;
- public RandomlySplittingBKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims,
+ public RandomlySplittingBKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDataDims, int numIndexDims,
int bytesPerDim, int maxPointsInLeafNode, double maxMBSortInHeap,
long totalPointCount, boolean singleValuePerDoc, int randomSeed) throws IOException {
- super(maxDoc, tempDir, tempFileNamePrefix, numDims, bytesPerDim, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount,
+ super(maxDoc, tempDir, tempFileNamePrefix, numDataDims, numIndexDims, bytesPerDim, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount,
getRandomSingleValuePerDoc(singleValuePerDoc, randomSeed),
getRandomLongOrds(totalPointCount, singleValuePerDoc, randomSeed),
getRandomOfflineSorterBufferMB(randomSeed),
@@ -291,7 +292,7 @@ public class RandomCodec extends AssertingCodec {
@Override
protected int split(byte[] minPackedValue, byte[] maxPackedValue, int[] parentDims) {
// BKD normally defaults by the widest dimension, to try to make as squarish cells as possible, but we just pick a random one ;)
- return random.nextInt(numDims);
+ return random.nextInt(numIndexDims);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
index 9f2d9b7..6da2761 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
@@ -130,7 +130,7 @@ public class RandomPostingsTester {
fieldInfoArray[fieldUpto] = new FieldInfo(field, fieldUpto, false, false, true,
IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
DocValuesType.NONE, -1, new HashMap<>(),
- 0, 0, false);
+ 0, 0, 0, false);
fieldUpto++;
SortedMap<BytesRef,SeedAndOrd> postings = new TreeMap<>();
@@ -651,7 +651,7 @@ public class RandomPostingsTester {
DocValuesType.NONE,
-1,
new HashMap<>(),
- 0, 0, false);
+ 0, 0, 0, false);
}
FieldInfos newFieldInfos = new FieldInfos(newFieldInfoArray);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
index 68e7190..95150fb 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
@@ -2627,10 +2627,12 @@ public abstract class LuceneTestCase extends Assert {
FieldInfos fieldInfos1 = MultiFields.getMergedFieldInfos(leftReader);
FieldInfos fieldInfos2 = MultiFields.getMergedFieldInfos(rightReader);
for(FieldInfo fieldInfo1 : fieldInfos1) {
- if (fieldInfo1.getPointDimensionCount() != 0) {
+ if (fieldInfo1.getPointDataDimensionCount() != 0) {
FieldInfo fieldInfo2 = fieldInfos2.fieldInfo(fieldInfo1.name);
- // same dimension count?
- assertEquals(info, fieldInfo2.getPointDimensionCount(), fieldInfo2.getPointDimensionCount());
+ // same data dimension count?
+ assertEquals(info, fieldInfo2.getPointDataDimensionCount(), fieldInfo2.getPointDataDimensionCount());
+ // same index dimension count?
+ assertEquals(info, fieldInfo2.getPointIndexDimensionCount(), fieldInfo2.getPointIndexDimensionCount());
// same bytes per dimension?
assertEquals(info, fieldInfo2.getPointNumBytes(), fieldInfo2.getPointNumBytes());
@@ -2642,10 +2644,12 @@ public abstract class LuceneTestCase extends Assert {
// make sure FieldInfos2 doesn't have any point fields that FieldInfo1 didn't have
for(FieldInfo fieldInfo2 : fieldInfos2) {
- if (fieldInfo2.getPointDimensionCount() != 0) {
+ if (fieldInfo2.getPointDataDimensionCount() != 0) {
FieldInfo fieldInfo1 = fieldInfos1.fieldInfo(fieldInfo2.name);
- // same dimension count?
- assertEquals(info, fieldInfo2.getPointDimensionCount(), fieldInfo1.getPointDimensionCount());
+ // same data dimension count?
+ assertEquals(info, fieldInfo2.getPointDataDimensionCount(), fieldInfo1.getPointDataDimensionCount());
+ // same index dimension count?
+ assertEquals(info, fieldInfo2.getPointIndexDimensionCount(), fieldInfo1.getPointIndexDimensionCount());
// same bytes per dimension?
assertEquals(info, fieldInfo2.getPointNumBytes(), fieldInfo1.getPointNumBytes());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
index 65329b5..1aa0300 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
@@ -1082,7 +1082,7 @@ public final class TestUtil {
final Field field1 = (Field) f;
final Field field2;
final DocValuesType dvType = field1.fieldType().docValuesType();
- final int dimCount = field1.fieldType().pointDimensionCount();
+ final int dimCount = field1.fieldType().pointDataDimensionCount();
if (dvType != DocValuesType.NONE) {
switch(dvType) {
case NUMERIC:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
index e79161d..2cbe703 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
@@ -795,7 +795,8 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
DocValuesType.NONE,
fieldInfo.getDocValuesGen(),
fieldInfo.attributes(),
- fieldInfo.getPointDimensionCount(),
+ fieldInfo.getPointDataDimensionCount(),
+ fieldInfo.getPointIndexDimensionCount(),
fieldInfo.getPointNumBytes(),
fieldInfo.isSoftDeletesField());
newInfos.add(f);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java b/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java
index 84f90ef..b2f239e 100644
--- a/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java
+++ b/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java
@@ -173,7 +173,7 @@ public class BBoxStrategy extends SpatialStrategy {
if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
numQuads++;
}
- if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
+ if ((this.hasPointVals = fieldType.pointDataDimensionCount() > 0)) {
numQuads++;
}
if (fieldType.indexOptions() != IndexOptions.NONE && fieldType instanceof LegacyFieldType && ((LegacyFieldType)fieldType).numericType() != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/solr/core/src/java/org/apache/solr/legacy/PointVectorStrategy.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/legacy/PointVectorStrategy.java b/solr/core/src/java/org/apache/solr/legacy/PointVectorStrategy.java
index da48fcb..4df76ca 100644
--- a/solr/core/src/java/org/apache/solr/legacy/PointVectorStrategy.java
+++ b/solr/core/src/java/org/apache/solr/legacy/PointVectorStrategy.java
@@ -152,7 +152,7 @@ public class PointVectorStrategy extends SpatialStrategy {
if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
numPairs++;
}
- if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
+ if ((this.hasPointVals = fieldType.pointDataDimensionCount() > 0)) {
numPairs++;
}
if (fieldType.indexOptions() != IndexOptions.NONE && fieldType instanceof LegacyFieldType && ((LegacyFieldType)fieldType).numericType() != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/solr/core/src/java/org/apache/solr/schema/SchemaField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaField.java b/solr/core/src/java/org/apache/solr/schema/SchemaField.java
index 256cbae..e28629e 100644
--- a/solr/core/src/java/org/apache/solr/schema/SchemaField.java
+++ b/solr/core/src/java/org/apache/solr/schema/SchemaField.java
@@ -415,7 +415,12 @@ public final class SchemaField extends FieldProperties implements IndexableField
}
@Override
- public int pointDimensionCount() {
+ public int pointDataDimensionCount() {
+ return 0;
+ }
+
+ @Override
+ public int pointIndexDimensionCount() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
index 8875afc..57d3d43 100644
--- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
@@ -431,7 +431,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
DocValuesType.NONE,
fieldInfo.getDocValuesGen(),
fieldInfo.attributes(),
- 0, 0, fieldInfo.isSoftDeletesField());
+ 0, 0, 0, fieldInfo.isSoftDeletesField());
newInfos.add(f);
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/solr/core/src/java/org/apache/solr/search/Insanity.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/Insanity.java b/solr/core/src/java/org/apache/solr/search/Insanity.java
index 67d1320..7da7a5c 100644
--- a/solr/core/src/java/org/apache/solr/search/Insanity.java
+++ b/solr/core/src/java/org/apache/solr/search/Insanity.java
@@ -67,7 +67,7 @@ public class Insanity {
if (fi.name.equals(insaneField)) {
filteredInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(),
fi.hasPayloads(), fi.getIndexOptions(), DocValuesType.NONE, -1, Collections.emptyMap(),
- fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()));
+ fi.getPointDataDimensionCount(), fi.getPointIndexDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()));
} else {
filteredInfos.add(fi);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java b/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java
index e6e8fda..42c34ab 100644
--- a/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java
+++ b/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java
@@ -592,11 +592,11 @@ public class FieldCacheImpl implements FieldCache {
if (parser instanceof PointParser) {
// points case
// no points in this segment
- if (info.getPointDimensionCount() == 0) {
+ if (info.getPointDataDimensionCount() == 0) {
return DocValues.emptyNumeric();
}
- if (info.getPointDimensionCount() != 1) {
- throw new IllegalStateException("Type mismatch: " + field + " was indexed with dimensions=" + info.getPointDimensionCount());
+ if (info.getPointDataDimensionCount() != 1) {
+ throw new IllegalStateException("Type mismatch: " + field + " was indexed with dimensions=" + info.getPointDataDimensionCount());
}
PointValues values = reader.getPointValues(field);
// no actual points for this field (e.g. all points deleted)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
index 176795d..e804635 100644
--- a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
+++ b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
@@ -237,12 +237,12 @@ public class UninvertingReader extends FilterLeafReader {
DocValuesType type = fi.getDocValuesType();
// fields which currently don't have docValues, but are uninvertable (indexed or points data present)
if (type == DocValuesType.NONE &&
- (fi.getIndexOptions() != IndexOptions.NONE || (fi.getPointNumBytes() > 0 && fi.getPointDimensionCount() == 1))) {
+ (fi.getIndexOptions() != IndexOptions.NONE || (fi.getPointNumBytes() > 0 && fi.getPointDataDimensionCount() == 1))) {
Type t = mapping.apply(fi.name); // could definitely return null, thus still can't uninvert it
if (t != null) {
if (t == Type.INTEGER_POINT || t == Type.LONG_POINT || t == Type.FLOAT_POINT || t == Type.DOUBLE_POINT) {
// type uses points
- if (fi.getPointDimensionCount() == 0) {
+ if (fi.getPointDataDimensionCount() == 0) {
continue;
}
} else {
@@ -284,7 +284,7 @@ public class UninvertingReader extends FilterLeafReader {
wrap = true;
newFieldInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(),
fi.hasPayloads(), fi.getIndexOptions(), type, fi.getDocValuesGen(), fi.attributes(),
- fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()));
+ fi.getPointDataDimensionCount(), fi.getPointIndexDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()));
} else {
newFieldInfos.add(fi);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java b/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java
index 9ec1234..05a1254 100644
--- a/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java
@@ -379,12 +379,14 @@ public class TestUninvertingReader extends LuceneTestCase {
FieldInfo intFInfo = fieldInfos.fieldInfo("int");
assertEquals(DocValuesType.NUMERIC, intFInfo.getDocValuesType());
- assertEquals(0, intFInfo.getPointDimensionCount());
+ assertEquals(0, intFInfo.getPointDataDimensionCount());
+ assertEquals(0, intFInfo.getPointIndexDimensionCount());
assertEquals(0, intFInfo.getPointNumBytes());
FieldInfo dintFInfo = fieldInfos.fieldInfo("dint");
assertEquals(DocValuesType.NUMERIC, dintFInfo.getDocValuesType());
- assertEquals(1, dintFInfo.getPointDimensionCount());
+ assertEquals(1, dintFInfo.getPointDataDimensionCount());
+ assertEquals(1, dintFInfo.getPointIndexDimensionCount());
assertEquals(4, dintFInfo.getPointNumBytes());
FieldInfo dvFInfo = fieldInfos.fieldInfo("dv");
[3/4] lucene-solr:master: LUCENE-8496: Selective indexing - modify
BKDReader/BKDWriter to allow users to select a fewer number of dimensions to
be used for creating the index than the total number of dimensions used for
field encoding. i.e., dimensions 0
Posted by nk...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java
index 773878a..f623701 100644
--- a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java
@@ -255,9 +255,9 @@ abstract class RangeFieldQuery extends Query {
/** Check indexed field info against the provided query data. */
private void checkFieldInfo(FieldInfo fieldInfo) {
- if (fieldInfo.getPointDimensionCount()/2 != numDims) {
+ if (fieldInfo.getPointDataDimensionCount()/2 != numDims) {
throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numDims="
- + fieldInfo.getPointDimensionCount()/2 + " but this query has numDims=" + numDims);
+ + fieldInfo.getPointDataDimensionCount()/2 + " but this query has numDims=" + numDims);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
index aa01723..aa799f2 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
@@ -1900,7 +1900,7 @@ public final class CheckIndex implements Closeable {
throw new RuntimeException("there are fields with points, but reader.getPointsReader() is null");
}
for (FieldInfo fieldInfo : fieldInfos) {
- if (fieldInfo.getPointDimensionCount() > 0) {
+ if (fieldInfo.getPointDataDimensionCount() > 0) {
PointValues values = pointsReader.getValues(fieldInfo.name);
if (values == null) {
continue;
@@ -1970,7 +1970,9 @@ public final class CheckIndex implements Closeable {
private final byte[] globalMinPackedValue;
private final byte[] globalMaxPackedValue;
private final int packedBytesCount;
- private final int numDims;
+ private final int packedIndexBytesCount;
+ private final int numDataDims;
+ private final int numIndexDims;
private final int bytesPerDim;
private final String fieldName;
@@ -1978,14 +1980,16 @@ public final class CheckIndex implements Closeable {
public VerifyPointsVisitor(String fieldName, int maxDoc, PointValues values) throws IOException {
this.maxDoc = maxDoc;
this.fieldName = fieldName;
- numDims = values.getNumDimensions();
+ numDataDims = values.getNumDataDimensions();
+ numIndexDims = values.getNumIndexDimensions();
bytesPerDim = values.getBytesPerDimension();
- packedBytesCount = numDims * bytesPerDim;
+ packedBytesCount = numDataDims * bytesPerDim;
+ packedIndexBytesCount = numIndexDims * bytesPerDim;
globalMinPackedValue = values.getMinPackedValue();
globalMaxPackedValue = values.getMaxPackedValue();
docsSeen = new FixedBitSet(maxDoc);
- lastMinPackedValue = new byte[packedBytesCount];
- lastMaxPackedValue = new byte[packedBytesCount];
+ lastMinPackedValue = new byte[packedIndexBytesCount];
+ lastMaxPackedValue = new byte[packedIndexBytesCount];
lastPackedValue = new byte[packedBytesCount];
if (values.getDocCount() > values.size()) {
@@ -2000,14 +2004,14 @@ public final class CheckIndex implements Closeable {
if (values.size() != 0) {
throw new RuntimeException("getMinPackedValue is null points for field \"" + fieldName + "\" yet size=" + values.size());
}
- } else if (globalMinPackedValue.length != packedBytesCount) {
+ } else if (globalMinPackedValue.length != packedIndexBytesCount) {
throw new RuntimeException("getMinPackedValue for field \"" + fieldName + "\" return length=" + globalMinPackedValue.length + " array, but should be " + packedBytesCount);
}
if (globalMaxPackedValue == null) {
if (values.size() != 0) {
throw new RuntimeException("getMaxPackedValue is null points for field \"" + fieldName + "\" yet size=" + values.size());
}
- } else if (globalMaxPackedValue.length != packedBytesCount) {
+ } else if (globalMaxPackedValue.length != packedIndexBytesCount) {
throw new RuntimeException("getMaxPackedValue for field \"" + fieldName + "\" return length=" + globalMaxPackedValue.length + " array, but should be " + packedBytesCount);
}
}
@@ -2033,7 +2037,7 @@ public final class CheckIndex implements Closeable {
pointCountSeen++;
docsSeen.set(docID);
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
int offset = bytesPerDim * dim;
// Compare to last cell:
@@ -2048,9 +2052,10 @@ public final class CheckIndex implements Closeable {
}
}
- // In the 1D case, PointValues must make a single in-order sweep through all values, and tie-break by
+ // In the 1D data case, PointValues must make a single in-order sweep through all values, and tie-break by
// increasing docID:
- if (numDims == 1) {
+ // for data dimension > 1, leaves are sorted by the dimension with the lowest cardinality to improve block compression
+ if (numDataDims == 1) {
int cmp = FutureArrays.compareUnsigned(lastPackedValue, 0, bytesPerDim, packedValue, 0, bytesPerDim);
if (cmp > 0) {
throw new RuntimeException("packed points value " + Arrays.toString(packedValue) + " for field=\"" + fieldName + "\", for docID=" + docID + " is out-of-order vs the previous document's value " + Arrays.toString(lastPackedValue));
@@ -2067,11 +2072,11 @@ public final class CheckIndex implements Closeable {
@Override
public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
checkPackedValue("min packed value", minPackedValue, -1);
- System.arraycopy(minPackedValue, 0, lastMinPackedValue, 0, packedBytesCount);
+ System.arraycopy(minPackedValue, 0, lastMinPackedValue, 0, packedIndexBytesCount);
checkPackedValue("max packed value", maxPackedValue, -1);
- System.arraycopy(maxPackedValue, 0, lastMaxPackedValue, 0, packedBytesCount);
+ System.arraycopy(maxPackedValue, 0, lastMaxPackedValue, 0, packedIndexBytesCount);
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
int offset = bytesPerDim * dim;
if (FutureArrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, maxPackedValue, offset, offset + bytesPerDim) > 0) {
@@ -2110,8 +2115,8 @@ public final class CheckIndex implements Closeable {
throw new RuntimeException(desc + " is null for docID=" + docID + " field=\"" + fieldName + "\"");
}
- if (packedValue.length != packedBytesCount) {
- throw new RuntimeException(desc + " has incorrect length=" + packedValue.length + " vs expected=" + packedBytesCount + " for docID=" + docID + " field=\"" + fieldName + "\"");
+ if (packedValue.length != (docID < 0 ? packedIndexBytesCount : packedBytesCount)) {
+ throw new RuntimeException(desc + " has incorrect length=" + packedValue.length + " vs expected=" + packedIndexBytesCount + " for docID=" + docID + " field=\"" + fieldName + "\"");
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/index/CodecReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/CodecReader.java b/lucene/core/src/java/org/apache/lucene/index/CodecReader.java
index 50aaa42..ca6ac7d 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CodecReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CodecReader.java
@@ -194,7 +194,7 @@ public abstract class CodecReader extends LeafReader implements Accountable {
public final PointValues getPointValues(String field) throws IOException {
ensureOpen();
FieldInfo fi = getFieldInfos().fieldInfo(field);
- if (fi == null || fi.getPointDimensionCount() == 0) {
+ if (fi == null || fi.getPointDataDimensionCount() == 0) {
// Field does not exist or does not index points
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
index d0a6974..4cc981d 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
@@ -201,7 +201,7 @@ final class DefaultIndexingChain extends DocConsumer {
PerField perField = fieldHash[i];
while (perField != null) {
if (perField.pointValuesWriter != null) {
- if (perField.fieldInfo.getPointDimensionCount() == 0) {
+ if (perField.fieldInfo.getPointDataDimensionCount() == 0) {
// BUG
throw new AssertionError("segment=" + state.segmentInfo + ": field=\"" + perField.fieldInfo.name + "\" has no points but wrote them");
}
@@ -216,7 +216,7 @@ final class DefaultIndexingChain extends DocConsumer {
perField.pointValuesWriter.flush(state, sortMap, pointsWriter);
perField.pointValuesWriter = null;
- } else if (perField.fieldInfo.getPointDimensionCount() != 0) {
+ } else if (perField.fieldInfo.getPointDataDimensionCount() != 0) {
// BUG
throw new AssertionError("segment=" + state.segmentInfo + ": field=\"" + perField.fieldInfo.name + "\" has points but did not write them");
}
@@ -477,7 +477,7 @@ final class DefaultIndexingChain extends DocConsumer {
}
indexDocValue(fp, dvType, field);
}
- if (fieldType.pointDimensionCount() != 0) {
+ if (fieldType.pointDataDimensionCount() != 0) {
if (fp == null) {
fp = getOrAddField(fieldName, fieldType, false);
}
@@ -508,17 +508,18 @@ final class DefaultIndexingChain extends DocConsumer {
/** Called from processDocument to index one field's point */
private void indexPoint(PerField fp, IndexableField field) throws IOException {
- int pointDimensionCount = field.fieldType().pointDimensionCount();
+ int pointDataDimensionCount = field.fieldType().pointDataDimensionCount();
+ int pointIndexDimensionCount = field.fieldType().pointIndexDimensionCount();
int dimensionNumBytes = field.fieldType().pointNumBytes();
// Record dimensions for this field; this setter will throw IllegalArgExc if
// the dimensions were already set to something different:
- if (fp.fieldInfo.getPointDimensionCount() == 0) {
- fieldInfos.globalFieldNumbers.setDimensions(fp.fieldInfo.number, fp.fieldInfo.name, pointDimensionCount, dimensionNumBytes);
+ if (fp.fieldInfo.getPointDataDimensionCount() == 0) {
+ fieldInfos.globalFieldNumbers.setDimensions(fp.fieldInfo.number, fp.fieldInfo.name, pointDataDimensionCount, pointIndexDimensionCount, dimensionNumBytes);
}
- fp.fieldInfo.setPointDimensions(pointDimensionCount, dimensionNumBytes);
+ fp.fieldInfo.setPointDimensions(pointDataDimensionCount, pointIndexDimensionCount, dimensionNumBytes);
if (fp.pointValuesWriter == null) {
fp.pointValuesWriter = new PointValuesWriter(docWriter, fp.fieldInfo);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
index b50cb12..c5d85bc 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
@@ -50,7 +50,8 @@ public final class FieldInfo {
/** If both of these are positive it means this field indexed points
* (see {@link org.apache.lucene.codecs.PointsFormat}). */
- private int pointDimensionCount;
+ private int pointDataDimensionCount;
+ private int pointIndexDimensionCount;
private int pointNumBytes;
// whether this field is used as the soft-deletes field
@@ -63,7 +64,7 @@ public final class FieldInfo {
*/
public FieldInfo(String name, int number, boolean storeTermVector, boolean omitNorms, boolean storePayloads,
IndexOptions indexOptions, DocValuesType docValues, long dvGen, Map<String,String> attributes,
- int pointDimensionCount, int pointNumBytes, boolean softDeletesField) {
+ int pointDataDimensionCount, int pointIndexDimensionCount, int pointNumBytes, boolean softDeletesField) {
this.name = Objects.requireNonNull(name);
this.number = number;
this.docValuesType = Objects.requireNonNull(docValues, "DocValuesType must not be null (field: \"" + name + "\")");
@@ -79,7 +80,8 @@ public final class FieldInfo {
}
this.dvGen = dvGen;
this.attributes = Objects.requireNonNull(attributes);
- this.pointDimensionCount = pointDimensionCount;
+ this.pointDataDimensionCount = pointDataDimensionCount;
+ this.pointIndexDimensionCount = pointIndexDimensionCount;
this.pointNumBytes = pointNumBytes;
this.softDeletesField = softDeletesField;
assert checkConsistency();
@@ -107,20 +109,28 @@ public final class FieldInfo {
}
}
- if (pointDimensionCount < 0) {
- throw new IllegalStateException("pointDimensionCount must be >= 0; got " + pointDimensionCount);
+ if (pointDataDimensionCount < 0) {
+ throw new IllegalStateException("pointDataDimensionCount must be >= 0; got " + pointDataDimensionCount);
+ }
+
+ if (pointIndexDimensionCount < 0) {
+ throw new IllegalStateException("pointIndexDimensionCount must be >= 0; got " + pointIndexDimensionCount);
}
if (pointNumBytes < 0) {
throw new IllegalStateException("pointNumBytes must be >= 0; got " + pointNumBytes);
}
- if (pointDimensionCount != 0 && pointNumBytes == 0) {
- throw new IllegalStateException("pointNumBytes must be > 0 when pointDimensionCount=" + pointDimensionCount);
+ if (pointDataDimensionCount != 0 && pointNumBytes == 0) {
+ throw new IllegalStateException("pointNumBytes must be > 0 when pointDataDimensionCount=" + pointDataDimensionCount);
+ }
+
+ if (pointIndexDimensionCount != 0 && pointDataDimensionCount == 0) {
+ throw new IllegalStateException("pointIndexDimensionCount must be 0 when pointDataDimensionCount=0");
}
- if (pointNumBytes != 0 && pointDimensionCount == 0) {
- throw new IllegalStateException("pointDimensionCount must be > 0 when pointNumBytes=" + pointNumBytes);
+ if (pointNumBytes != 0 && pointDataDimensionCount == 0) {
+ throw new IllegalStateException("pointDataDimensionCount must be > 0 when pointNumBytes=" + pointNumBytes);
}
if (dvGen != -1 && docValuesType == DocValuesType.NONE) {
@@ -132,7 +142,7 @@ public final class FieldInfo {
// should only be called by FieldInfos#addOrUpdate
void update(boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions,
- int dimensionCount, int dimensionNumBytes) {
+ int dataDimensionCount, int indexDimensionCount, int dimensionNumBytes) {
if (indexOptions == null) {
throw new NullPointerException("IndexOptions must not be null (field: \"" + name + "\")");
}
@@ -145,11 +155,12 @@ public final class FieldInfo {
}
}
- if (this.pointDimensionCount == 0 && dimensionCount != 0) {
- this.pointDimensionCount = dimensionCount;
+ if (this.pointDataDimensionCount == 0 && dataDimensionCount != 0) {
+ this.pointDataDimensionCount = dataDimensionCount;
+ this.pointIndexDimensionCount = indexDimensionCount;
this.pointNumBytes = dimensionNumBytes;
- } else if (dimensionCount != 0 && (this.pointDimensionCount != dimensionCount || this.pointNumBytes != dimensionNumBytes)) {
- throw new IllegalArgumentException("cannot change field \"" + name + "\" from points dimensionCount=" + this.pointDimensionCount + ", numBytes=" + this.pointNumBytes + " to inconsistent dimensionCount=" + dimensionCount + ", numBytes=" + dimensionNumBytes);
+ } else if (dataDimensionCount != 0 && (this.pointDataDimensionCount != dataDimensionCount || this.pointIndexDimensionCount != indexDimensionCount || this.pointNumBytes != dimensionNumBytes)) {
+ throw new IllegalArgumentException("cannot change field \"" + name + "\" from points dataDimensionCount=" + this.pointDataDimensionCount + ", indexDimensionCount=" + this.pointIndexDimensionCount + ", numBytes=" + this.pointNumBytes + " to inconsistent dataDimensionCount=" + dataDimensionCount +", indexDimensionCount=" + indexDimensionCount + ", numBytes=" + dimensionNumBytes);
}
if (this.indexOptions != IndexOptions.NONE) { // if updated field data is not for indexing, leave the updates out
@@ -170,12 +181,15 @@ public final class FieldInfo {
/** Record that this field is indexed with points, with the
* specified number of dimensions and bytes per dimension. */
- public void setPointDimensions(int count, int numBytes) {
- if (count <= 0) {
- throw new IllegalArgumentException("point dimension count must be >= 0; got " + count + " for field=\"" + name + "\"");
+ public void setPointDimensions(int dataDimensionCount, int indexDimensionCount, int numBytes) {
+ if (dataDimensionCount <= 0) {
+ throw new IllegalArgumentException("point data dimension count must be >= 0; got " + dataDimensionCount + " for field=\"" + name + "\"");
}
- if (count > PointValues.MAX_DIMENSIONS) {
- throw new IllegalArgumentException("point dimension count must be < PointValues.MAX_DIMENSIONS (= " + PointValues.MAX_DIMENSIONS + "); got " + count + " for field=\"" + name + "\"");
+ if (dataDimensionCount > PointValues.MAX_DIMENSIONS) {
+ throw new IllegalArgumentException("point data dimension count must be < PointValues.MAX_DIMENSIONS (= " + PointValues.MAX_DIMENSIONS + "); got " + dataDimensionCount + " for field=\"" + name + "\"");
+ }
+ if (indexDimensionCount > dataDimensionCount) {
+ throw new IllegalArgumentException("point index dimension count must be <= point data dimension count (= " + dataDimensionCount + "); got " + indexDimensionCount + " for field=\"" + name + "\"");
}
if (numBytes <= 0) {
throw new IllegalArgumentException("point numBytes must be >= 0; got " + numBytes + " for field=\"" + name + "\"");
@@ -183,22 +197,31 @@ public final class FieldInfo {
if (numBytes > PointValues.MAX_NUM_BYTES) {
throw new IllegalArgumentException("point numBytes must be <= PointValues.MAX_NUM_BYTES (= " + PointValues.MAX_NUM_BYTES + "); got " + numBytes + " for field=\"" + name + "\"");
}
- if (pointDimensionCount != 0 && pointDimensionCount != count) {
- throw new IllegalArgumentException("cannot change point dimension count from " + pointDimensionCount + " to " + count + " for field=\"" + name + "\"");
+ if (pointDataDimensionCount != 0 && pointDataDimensionCount != dataDimensionCount) {
+ throw new IllegalArgumentException("cannot change point data dimension count from " + pointDataDimensionCount + " to " + dataDimensionCount + " for field=\"" + name + "\"");
+ }
+ if (pointIndexDimensionCount != 0 && pointIndexDimensionCount != indexDimensionCount) {
+ throw new IllegalArgumentException("cannot change point index dimension count from " + pointIndexDimensionCount + " to " + indexDimensionCount + " for field=\"" + name + "\"");
}
if (pointNumBytes != 0 && pointNumBytes != numBytes) {
throw new IllegalArgumentException("cannot change point numBytes from " + pointNumBytes + " to " + numBytes + " for field=\"" + name + "\"");
}
- pointDimensionCount = count;
+ pointDataDimensionCount = dataDimensionCount;
+ pointIndexDimensionCount = indexDimensionCount;
pointNumBytes = numBytes;
assert checkConsistency();
}
- /** Return point dimension count */
- public int getPointDimensionCount() {
- return pointDimensionCount;
+ /** Return point data dimension count */
+ public int getPointDataDimensionCount() {
+ return pointDataDimensionCount;
+ }
+
+ /** Return point data dimension count */
+ public int getPointIndexDimensionCount() {
+ return pointIndexDimensionCount;
}
/** Return number of bytes per dimension */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
index 0a0ff5e..1e0ef94 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
@@ -93,7 +93,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
hasNorms |= info.hasNorms();
hasDocValues |= info.getDocValuesType() != DocValuesType.NONE;
hasPayloads |= info.hasPayloads();
- hasPointValues |= (info.getPointDimensionCount() != 0);
+ hasPointValues |= (info.getPointDataDimensionCount() != 0);
if (info.isSoftDeletesField()) {
if (softDeletesField != null && softDeletesField.equals(info.name) == false) {
throw new IllegalArgumentException("multiple soft-deletes fields [" + info.name + ", " + softDeletesField + "]");
@@ -210,11 +210,13 @@ public class FieldInfos implements Iterable<FieldInfo> {
}
static final class FieldDimensions {
- public final int dimensionCount;
+ public final int dataDimensionCount;
+ public final int indexDimensionCount;
public final int dimensionNumBytes;
- public FieldDimensions(int dimensionCount, int dimensionNumBytes) {
- this.dimensionCount = dimensionCount;
+ public FieldDimensions(int dataDimensionCount, int indexDimensionCount, int dimensionNumBytes) {
+ this.dataDimensionCount = dataDimensionCount;
+ this.indexDimensionCount = indexDimensionCount;
this.dimensionNumBytes = dimensionNumBytes;
}
}
@@ -254,7 +256,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
* number assigned if possible otherwise the first unassigned field number
* is used as the field number.
*/
- synchronized int addOrGet(String fieldName, int preferredFieldNumber, IndexOptions indexOptions, DocValuesType dvType, int dimensionCount, int dimensionNumBytes, boolean isSoftDeletesField) {
+ synchronized int addOrGet(String fieldName, int preferredFieldNumber, IndexOptions indexOptions, DocValuesType dvType, int dataDimensionCount, int indexDimensionCount, int dimensionNumBytes, boolean isSoftDeletesField) {
if (indexOptions != IndexOptions.NONE) {
IndexOptions currentOpts = this.indexOptions.get(fieldName);
if (currentOpts == null) {
@@ -271,17 +273,20 @@ public class FieldInfos implements Iterable<FieldInfo> {
throw new IllegalArgumentException("cannot change DocValues type from " + currentDVType + " to " + dvType + " for field \"" + fieldName + "\"");
}
}
- if (dimensionCount != 0) {
+ if (dataDimensionCount != 0) {
FieldDimensions dims = dimensions.get(fieldName);
if (dims != null) {
- if (dims.dimensionCount != dimensionCount) {
- throw new IllegalArgumentException("cannot change point dimension count from " + dims.dimensionCount + " to " + dimensionCount + " for field=\"" + fieldName + "\"");
+ if (dims.dataDimensionCount != dataDimensionCount) {
+ throw new IllegalArgumentException("cannot change point data dimension count from " + dims.dataDimensionCount + " to " + dataDimensionCount + " for field=\"" + fieldName + "\"");
+ }
+ if (dims.indexDimensionCount != indexDimensionCount) {
+ throw new IllegalArgumentException("cannot change point index dimension count from " + dims.indexDimensionCount + " to " + indexDimensionCount + " for field=\"" + fieldName + "\"");
}
if (dims.dimensionNumBytes != dimensionNumBytes) {
throw new IllegalArgumentException("cannot change point numBytes from " + dims.dimensionNumBytes + " to " + dimensionNumBytes + " for field=\"" + fieldName + "\"");
}
} else {
- dimensions.put(fieldName, new FieldDimensions(dimensionCount, dimensionNumBytes));
+ dimensions.put(fieldName, new FieldDimensions(dataDimensionCount, indexDimensionCount, dimensionNumBytes));
}
}
Integer fieldNumber = nameToNumber.get(fieldName);
@@ -341,7 +346,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
}
}
- synchronized void verifyConsistentDimensions(Integer number, String name, int dimensionCount, int dimensionNumBytes) {
+ synchronized void verifyConsistentDimensions(Integer number, String name, int dataDimensionCount, int indexDimensionCount, int dimensionNumBytes) {
if (name.equals(numberToName.get(number)) == false) {
throw new IllegalArgumentException("field number " + number + " is already mapped to field name \"" + numberToName.get(number) + "\", not \"" + name + "\"");
}
@@ -350,8 +355,11 @@ public class FieldInfos implements Iterable<FieldInfo> {
}
FieldDimensions dim = dimensions.get(name);
if (dim != null) {
- if (dim.dimensionCount != dimensionCount) {
- throw new IllegalArgumentException("cannot change point dimension count from " + dim.dimensionCount + " to " + dimensionCount + " for field=\"" + name + "\"");
+ if (dim.dataDimensionCount != dataDimensionCount) {
+ throw new IllegalArgumentException("cannot change point data dimension count from " + dim.dataDimensionCount + " to " + dataDimensionCount + " for field=\"" + name + "\"");
+ }
+ if (dim.indexDimensionCount != indexDimensionCount) {
+ throw new IllegalArgumentException("cannot change point index dimension count from " + dim.indexDimensionCount + " to " + indexDimensionCount + " for field=\"" + name + "\"");
}
if (dim.dimensionNumBytes != dimensionNumBytes) {
throw new IllegalArgumentException("cannot change point numBytes from " + dim.dimensionNumBytes + " to " + dimensionNumBytes + " for field=\"" + name + "\"");
@@ -395,15 +403,18 @@ public class FieldInfos implements Iterable<FieldInfo> {
docValuesType.put(name, dvType);
}
- synchronized void setDimensions(int number, String name, int dimensionCount, int dimensionNumBytes) {
+ synchronized void setDimensions(int number, String name, int dataDimensionCount, int indexDimensionCount, int dimensionNumBytes) {
if (dimensionNumBytes > PointValues.MAX_NUM_BYTES) {
throw new IllegalArgumentException("dimension numBytes must be <= PointValues.MAX_NUM_BYTES (= " + PointValues.MAX_NUM_BYTES + "); got " + dimensionNumBytes + " for field=\"" + name + "\"");
}
- if (dimensionCount > PointValues.MAX_DIMENSIONS) {
- throw new IllegalArgumentException("pointDimensionCount must be <= PointValues.MAX_DIMENSIONS (= " + PointValues.MAX_DIMENSIONS + "); got " + dimensionCount + " for field=\"" + name + "\"");
+ if (dataDimensionCount > PointValues.MAX_DIMENSIONS) {
+ throw new IllegalArgumentException("pointDataDimensionCount must be <= PointValues.MAX_DIMENSIONS (= " + PointValues.MAX_DIMENSIONS + "); got " + dataDimensionCount + " for field=\"" + name + "\"");
+ }
+ if (indexDimensionCount > dataDimensionCount) {
+ throw new IllegalArgumentException("pointIndexDimensionCount must be <= pointDataDimensionCount (= " + dataDimensionCount + "); got " + indexDimensionCount + " for field=\"" + name + "\"");
}
- verifyConsistentDimensions(number, name, dimensionCount, dimensionNumBytes);
- dimensions.put(name, new FieldDimensions(dimensionCount, dimensionNumBytes));
+ verifyConsistentDimensions(number, name, dataDimensionCount, indexDimensionCount, dimensionNumBytes);
+ dimensions.put(name, new FieldDimensions(dataDimensionCount, indexDimensionCount, dimensionNumBytes));
}
}
@@ -438,8 +449,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
// before then we'll get the same name and number,
// else we'll allocate a new one:
final boolean isSoftDeletesField = name.equals(globalFieldNumbers.softDeletesFieldName);
- final int fieldNumber = globalFieldNumbers.addOrGet(name, -1, IndexOptions.NONE, DocValuesType.NONE, 0, 0, isSoftDeletesField);
- fi = new FieldInfo(name, fieldNumber, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, new HashMap<>(), 0, 0, isSoftDeletesField);
+ final int fieldNumber = globalFieldNumbers.addOrGet(name, -1, IndexOptions.NONE, DocValuesType.NONE, 0, 0, 0, isSoftDeletesField);
+ fi = new FieldInfo(name, fieldNumber, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, new HashMap<>(), 0, 0, 0, isSoftDeletesField);
assert !byName.containsKey(fi.name);
globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, DocValuesType.NONE);
byName.put(fi.name, fi);
@@ -452,7 +463,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
boolean storeTermVector,
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions,
DocValuesType docValues, long dvGen,
- int dimensionCount, int dimensionNumBytes, boolean isSoftDeletesField) {
+ int dataDimensionCount, int indexDimensionCount, int dimensionNumBytes,
+ boolean isSoftDeletesField) {
assert assertNotFinished();
if (docValues == null) {
throw new NullPointerException("DocValuesType must not be null");
@@ -464,13 +476,13 @@ public class FieldInfos implements Iterable<FieldInfo> {
// number for this field. If the field was seen
// before then we'll get the same name and number,
// else we'll allocate a new one:
- final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, indexOptions, docValues, dimensionCount, dimensionNumBytes, isSoftDeletesField);
- fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, dvGen, new HashMap<>(), dimensionCount, dimensionNumBytes, isSoftDeletesField);
+ final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, indexOptions, docValues, dataDimensionCount, indexDimensionCount, dimensionNumBytes, isSoftDeletesField);
+ fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, dvGen, new HashMap<>(), dataDimensionCount, indexDimensionCount, dimensionNumBytes, isSoftDeletesField);
assert !byName.containsKey(fi.name);
globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());
byName.put(fi.name, fi);
} else {
- fi.update(storeTermVector, omitNorms, storePayloads, indexOptions, dimensionCount, dimensionNumBytes);
+ fi.update(storeTermVector, omitNorms, storePayloads, indexOptions, dataDimensionCount, indexDimensionCount, dimensionNumBytes);
if (docValues != DocValuesType.NONE) {
// Only pay the synchronization cost if fi does not already have a DVType
@@ -498,7 +510,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
return addOrUpdateInternal(fi.name, fi.number, fi.hasVectors(),
fi.omitsNorms(), fi.hasPayloads(),
fi.getIndexOptions(), fi.getDocValuesType(), dvGen,
- fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField());
+ fi.getPointDataDimensionCount(), fi.getPointIndexDimensionCount(), fi.getPointNumBytes(),
+ fi.isSoftDeletesField());
}
public FieldInfo fieldInfo(String fieldName) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index db6ef9f..34424af 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -991,7 +991,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
for(SegmentCommitInfo info : segmentInfos) {
FieldInfos fis = readFieldInfos(info);
for(FieldInfo fi : fis) {
- map.addOrGet(fi.name, fi.number, fi.getIndexOptions(), fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField());
+ map.addOrGet(fi.name, fi.number, fi.getIndexOptions(), fi.getDocValuesType(), fi.getPointDataDimensionCount(), fi.getPointIndexDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField());
}
}
@@ -1813,7 +1813,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
if (globalFieldNumberMap.contains(f.name(), dvType) == false) {
// if this field doesn't exists we try to add it. if it exists and the DV type doesn't match we
// get a consistent error message as if you try to do that during an indexing operation.
- globalFieldNumberMap.addOrGet(f.name(), -1, IndexOptions.NONE, dvType, 0, 0, f.name().equals(config.softDeletesField));
+ globalFieldNumberMap.addOrGet(f.name(), -1, IndexOptions.NONE, dvType, 0, 0, 0, f.name().equals(config.softDeletesField));
assert globalFieldNumberMap.contains(f.name(), dvType);
}
if (config.getIndexSortFields().contains(f.name())) {
@@ -2849,7 +2849,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
FieldInfos fis = readFieldInfos(info);
for(FieldInfo fi : fis) {
// This will throw exceptions if any of the incoming fields have an illegal schema change:
- globalFieldNumberMap.addOrGet(fi.name, fi.number, fi.getIndexOptions(), fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField());
+ globalFieldNumberMap.addOrGet(fi.name, fi.number, fi.getIndexOptions(), fi.getDocValuesType(), fi.getPointDataDimensionCount(), fi.getPointIndexDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField());
}
infos.add(copySegmentAsIs(info, newSegName, context));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java b/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
index a797cf8..b2b2e77 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
@@ -98,9 +98,14 @@ public interface IndexableFieldType {
public DocValuesType docValuesType();
/**
- * If this is positive, the field is indexed as a point.
+ * If this is positive (representing the number of point data dimensions), the field is indexed as a point.
*/
- public int pointDimensionCount();
+ public int pointDataDimensionCount();
+
+ /**
+ * The number of dimensions used for the index key
+ */
+ public int pointIndexDimensionCount();
/**
* The number of bytes in each dimension's values.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/index/PointValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValues.java b/lucene/core/src/java/org/apache/lucene/index/PointValues.java
index df433d2..51b12d1 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PointValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PointValues.java
@@ -136,7 +136,7 @@ public abstract class PointValues {
if (minValue == null) {
minValue = leafMinValue.clone();
} else {
- final int numDimensions = values.getNumDimensions();
+ final int numDimensions = values.getNumIndexDimensions();
final int numBytesPerDimension = values.getBytesPerDimension();
for (int i = 0; i < numDimensions; ++i) {
int offset = i * numBytesPerDimension;
@@ -167,7 +167,7 @@ public abstract class PointValues {
if (maxValue == null) {
maxValue = leafMaxValue.clone();
} else {
- final int numDimensions = values.getNumDimensions();
+ final int numDimensions = values.getNumIndexDimensions();
final int numBytesPerDimension = values.getBytesPerDimension();
for (int i = 0; i < numDimensions; ++i) {
int offset = i * numBytesPerDimension;
@@ -233,8 +233,11 @@ public abstract class PointValues {
/** Returns maximum value for each dimension, packed, or null if {@link #size} is <code>0</code> */
public abstract byte[] getMaxPackedValue() throws IOException;
- /** Returns how many dimensions were indexed */
- public abstract int getNumDimensions() throws IOException;
+ /** Returns how many data dimensions are represented in the values */
+ public abstract int getNumDataDimensions() throws IOException;
+
+ /** Returns how many dimensions are used for the index */
+ public abstract int getNumIndexDimensions() throws IOException;
/** Returns the number of bytes per dimension */
public abstract int getBytesPerDimension() throws IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
index 4aaf095..eb11ec3 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
@@ -43,7 +43,7 @@ class PointValuesWriter {
this.bytes = new ByteBlockPool(docWriter.byteBlockAllocator);
docIDs = new int[16];
iwBytesUsed.addAndGet(16 * Integer.BYTES);
- packedBytesLength = fieldInfo.getPointDimensionCount() * fieldInfo.getPointNumBytes();
+ packedBytesLength = fieldInfo.getPointDataDimensionCount() * fieldInfo.getPointNumBytes();
}
// TODO: if exactly the same value is added to exactly the same doc, should we dedup?
@@ -52,7 +52,7 @@ class PointValuesWriter {
throw new IllegalArgumentException("field=" + fieldInfo.name + ": point value must not be null");
}
if (value.length != packedBytesLength) {
- throw new IllegalArgumentException("field=" + fieldInfo.name + ": this field's value has length=" + value.length + " but should be " + (fieldInfo.getPointDimensionCount() * fieldInfo.getPointNumBytes()));
+ throw new IllegalArgumentException("field=" + fieldInfo.name + ": this field's value has length=" + value.length + " but should be " + (fieldInfo.getPointDataDimensionCount() * fieldInfo.getPointNumBytes()));
}
if (docIDs.length == numPoints) {
@@ -106,7 +106,12 @@ class PointValuesWriter {
}
@Override
- public int getNumDimensions() {
+ public int getNumDataDimensions() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getNumIndexDimensions() {
throw new UnsupportedOperationException();
}
@@ -229,8 +234,13 @@ class PointValuesWriter {
}
@Override
- public int getNumDimensions() throws IOException {
- return in.getNumDimensions();
+ public int getNumDataDimensions() throws IOException {
+ return in.getNumDataDimensions();
+ }
+
+ @Override
+ public int getNumIndexDimensions() throws IOException {
+ return in.getNumIndexDimensions();
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java
index 55e4d20..d9ed94b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java
@@ -344,8 +344,13 @@ class SortingLeafReader extends FilterLeafReader {
}
@Override
- public int getNumDimensions() throws IOException {
- return in.getNumDimensions();
+ public int getNumDataDimensions() throws IOException {
+ return in.getNumDataDimensions();
+ }
+
+ @Override
+ public int getNumIndexDimensions() throws IOException {
+ return in.getNumIndexDimensions();
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java
index eca38a3..487bdfb 100644
--- a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java
@@ -123,8 +123,8 @@ public abstract class PointInSetQuery extends Query {
return null;
}
- if (values.getNumDimensions() != numDims) {
- throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numDims=" + values.getNumDimensions() + " but this query has numDims=" + numDims);
+ if (values.getNumIndexDimensions() != numDims) {
+ throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numIndexDims=" + values.getNumIndexDimensions() + " but this query has numIndexDims=" + numDims);
}
if (values.getBytesPerDimension() != bytesPerDim) {
throw new IllegalArgumentException("field=\"" + field + "\" was indexed with bytesPerDim=" + values.getBytesPerDimension() + " but this query has bytesPerDim=" + bytesPerDim);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java
index 031ca38..688a31f 100644
--- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java
@@ -234,8 +234,8 @@ public abstract class PointRangeQuery extends Query {
return null;
}
- if (values.getNumDimensions() != numDims) {
- throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numDims=" + values.getNumDimensions() + " but this query has numDims=" + numDims);
+ if (values.getNumIndexDimensions() != numDims) {
+ throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numIndexDimensions=" + values.getNumIndexDimensions() + " but this query has numDims=" + numDims);
}
if (bytesPerDim != values.getBytesPerDimension()) {
throw new IllegalArgumentException("field=\"" + field + "\" was indexed with bytesPerDim=" + values.getBytesPerDimension() + " but this query has bytesPerDim=" + bytesPerDim);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
index 00a1d7d..3cbb054 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
@@ -35,7 +35,8 @@ import org.apache.lucene.util.MathUtil;
public final class BKDReader extends PointValues implements Accountable {
// Packed array of byte[] holding all split values in the full binary tree:
final int leafNodeOffset;
- final int numDims;
+ final int numDataDims;
+ final int numIndexDims;
final int bytesPerDim;
final int numLeaves;
final IndexInput in;
@@ -46,29 +47,36 @@ public final class BKDReader extends PointValues implements Accountable {
final int docCount;
final int version;
protected final int packedBytesLength;
+ protected final int packedIndexBytesLength;
final byte[] packedIndex;
/** Caller must pre-seek the provided {@link IndexInput} to the index location that {@link BKDWriter#finish} returned */
public BKDReader(IndexInput in) throws IOException {
version = CodecUtil.checkHeader(in, BKDWriter.CODEC_NAME, BKDWriter.VERSION_START, BKDWriter.VERSION_CURRENT);
- numDims = in.readVInt();
+ numDataDims = in.readVInt();
+ if (version >= BKDWriter.VERSION_SELECTIVE_INDEXING) {
+ numIndexDims = in.readVInt();
+ } else {
+ numIndexDims = numDataDims;
+ }
maxPointsInLeafNode = in.readVInt();
bytesPerDim = in.readVInt();
- packedBytesLength = numDims * bytesPerDim;
+ packedBytesLength = numDataDims * bytesPerDim;
+ packedIndexBytesLength = numIndexDims * bytesPerDim;
// Read index:
numLeaves = in.readVInt();
assert numLeaves > 0;
leafNodeOffset = numLeaves;
- minPackedValue = new byte[packedBytesLength];
- maxPackedValue = new byte[packedBytesLength];
+ minPackedValue = new byte[packedIndexBytesLength];
+ maxPackedValue = new byte[packedIndexBytesLength];
- in.readBytes(minPackedValue, 0, packedBytesLength);
- in.readBytes(maxPackedValue, 0, packedBytesLength);
+ in.readBytes(minPackedValue, 0, packedIndexBytesLength);
+ in.readBytes(maxPackedValue, 0, packedIndexBytesLength);
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
if (FutureArrays.compareUnsigned(minPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim, maxPackedValue, dim * bytesPerDim, dim * bytesPerDim + bytesPerDim) > 0) {
throw new CorruptIndexException("minPackedValue " + new BytesRef(minPackedValue) + " is > maxPackedValue " + new BytesRef(maxPackedValue) + " for dim=" + dim, in);
}
@@ -122,16 +130,16 @@ public final class BKDReader extends PointValues implements Accountable {
splitPackedValueStack = new byte[treeDepth+1][];
nodeID = 1;
level = 1;
- splitPackedValueStack[level] = new byte[packedBytesLength];
+ splitPackedValueStack[level] = new byte[packedIndexBytesLength];
leafBlockFPStack = new long[treeDepth+1];
leftNodePositions = new int[treeDepth+1];
rightNodePositions = new int[treeDepth+1];
splitValuesStack = new byte[treeDepth+1][];
splitDims = new int[treeDepth+1];
- negativeDeltas = new boolean[numDims*(treeDepth+1)];
+ negativeDeltas = new boolean[numIndexDims*(treeDepth+1)];
in = new ByteArrayDataInput(packedIndex);
- splitValuesStack[0] = new byte[packedBytesLength];
+ splitValuesStack[0] = new byte[packedIndexBytesLength];
readNodeData(false);
scratch = new BytesRef();
scratch.length = bytesPerDim;
@@ -142,11 +150,11 @@ public final class BKDReader extends PointValues implements Accountable {
nodeID *= 2;
level++;
if (splitPackedValueStack[level] == null) {
- splitPackedValueStack[level] = new byte[packedBytesLength];
+ splitPackedValueStack[level] = new byte[packedIndexBytesLength];
}
- System.arraycopy(negativeDeltas, (level-1)*numDims, negativeDeltas, level*numDims, numDims);
+ System.arraycopy(negativeDeltas, (level-1)*numIndexDims, negativeDeltas, level*numIndexDims, numIndexDims);
assert splitDim != -1;
- negativeDeltas[level*numDims+splitDim] = true;
+ negativeDeltas[level*numIndexDims+splitDim] = true;
in.setPosition(nodePosition);
readNodeData(true);
}
@@ -162,7 +170,7 @@ public final class BKDReader extends PointValues implements Accountable {
index.leftNodePositions[level] = leftNodePositions[level];
index.rightNodePositions[level] = rightNodePositions[level];
index.splitValuesStack[index.level] = splitValuesStack[index.level].clone();
- System.arraycopy(negativeDeltas, level*numDims, index.negativeDeltas, level*numDims, numDims);
+ System.arraycopy(negativeDeltas, level*numIndexDims, index.negativeDeltas, level*numIndexDims, numIndexDims);
index.splitDims[level] = splitDims[level];
return index;
}
@@ -172,11 +180,11 @@ public final class BKDReader extends PointValues implements Accountable {
nodeID = nodeID * 2 + 1;
level++;
if (splitPackedValueStack[level] == null) {
- splitPackedValueStack[level] = new byte[packedBytesLength];
+ splitPackedValueStack[level] = new byte[packedIndexBytesLength];
}
- System.arraycopy(negativeDeltas, (level-1)*numDims, negativeDeltas, level*numDims, numDims);
+ System.arraycopy(negativeDeltas, (level-1)*numIndexDims, negativeDeltas, level*numIndexDims, numIndexDims);
assert splitDim != -1;
- negativeDeltas[level*numDims+splitDim] = false;
+ negativeDeltas[level*numIndexDims+splitDim] = false;
in.setPosition(nodePosition);
readNodeData(false);
}
@@ -276,19 +284,19 @@ public final class BKDReader extends PointValues implements Accountable {
// read split dim, prefix, firstDiffByteDelta encoded as int:
int code = in.readVInt();
- splitDim = code % numDims;
+ splitDim = code % numIndexDims;
splitDims[level] = splitDim;
- code /= numDims;
+ code /= numIndexDims;
int prefix = code % (1+bytesPerDim);
int suffix = bytesPerDim - prefix;
if (splitValuesStack[level] == null) {
- splitValuesStack[level] = new byte[packedBytesLength];
+ splitValuesStack[level] = new byte[packedIndexBytesLength];
}
- System.arraycopy(splitValuesStack[level-1], 0, splitValuesStack[level], 0, packedBytesLength);
+ System.arraycopy(splitValuesStack[level-1], 0, splitValuesStack[level], 0, packedIndexBytesLength);
if (suffix > 0) {
int firstDiffByteDelta = code / (1+bytesPerDim);
- if (negativeDeltas[level*numDims + splitDim]) {
+ if (negativeDeltas[level*numIndexDims + splitDim]) {
firstDiffByteDelta = -firstDiffByteDelta;
}
int oldByte = splitValuesStack[level][splitDim*bytesPerDim+prefix] & 0xFF;
@@ -325,7 +333,7 @@ public final class BKDReader extends PointValues implements Accountable {
public static final class IntersectState {
final IndexInput in;
final int[] scratchDocIDs;
- final byte[] scratchPackedValue1, scratchPackedValue2;
+ final byte[] scratchDataPackedValue, scratchMinIndexPackedValue, scratchMaxIndexPackedValue;
final int[] commonPrefixLengths;
final IntersectVisitor visitor;
@@ -333,6 +341,7 @@ public final class BKDReader extends PointValues implements Accountable {
public IntersectState(IndexInput in, int numDims,
int packedBytesLength,
+ int packedIndexBytesLength,
int maxPointsInLeafNode,
IntersectVisitor visitor,
IndexTree indexVisitor) {
@@ -340,8 +349,9 @@ public final class BKDReader extends PointValues implements Accountable {
this.visitor = visitor;
this.commonPrefixLengths = new int[numDims];
this.scratchDocIDs = new int[maxPointsInLeafNode];
- this.scratchPackedValue1 = new byte[packedBytesLength];
- this.scratchPackedValue2 = new byte[packedBytesLength];
+ this.scratchDataPackedValue = new byte[packedBytesLength];
+ this.scratchMinIndexPackedValue = new byte[packedIndexBytesLength];
+ this.scratchMaxIndexPackedValue = new byte[packedIndexBytesLength];
this.index = indexVisitor;
}
}
@@ -389,8 +399,9 @@ public final class BKDReader extends PointValues implements Accountable {
/** Create a new {@link IntersectState} */
public IntersectState getIntersectState(IntersectVisitor visitor) {
IndexTree index = new IndexTree();
- return new IntersectState(in.clone(), numDims,
+ return new IntersectState(in.clone(), numDataDims,
packedBytesLength,
+ packedIndexBytesLength,
maxPointsInLeafNode,
visitor,
index);
@@ -403,7 +414,7 @@ public final class BKDReader extends PointValues implements Accountable {
int count = readDocIDs(state.in, index.getLeafBlockFP(), state.scratchDocIDs);
// Again, this time reading values and checking with the visitor
- visitDocValues(state.commonPrefixLengths, state.scratchPackedValue1, state.scratchPackedValue2, state.in, state.scratchDocIDs, count, state.visitor);
+ visitDocValues(state.commonPrefixLengths, state.scratchDataPackedValue, state.scratchMinIndexPackedValue, state.scratchMaxIndexPackedValue, state.in, state.scratchDocIDs, count, state.visitor);
}
private void visitDocIDs(IndexInput in, long blockFP, IntersectVisitor visitor) throws IOException {
@@ -428,17 +439,19 @@ public final class BKDReader extends PointValues implements Accountable {
return count;
}
- void visitDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue1, byte[] scratchPackedValue2, IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException {
+ void visitDocValues(int[] commonPrefixLengths, byte[] scratchDataPackedValue, byte[] scratchMinIndexPackedValue, byte[] scratchMaxIndexPackedValue,
+ IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException {
- readCommonPrefixes(commonPrefixLengths, scratchPackedValue1, in);
+ readCommonPrefixes(commonPrefixLengths, scratchDataPackedValue, in);
- if (numDims != 1 && version >= BKDWriter.VERSION_LEAF_STORES_BOUNDS) {
- byte[] minPackedValue = scratchPackedValue1;
- byte[] maxPackedValue = scratchPackedValue2;
+ if (numIndexDims != 1 && version >= BKDWriter.VERSION_LEAF_STORES_BOUNDS) {
+ byte[] minPackedValue = scratchMinIndexPackedValue;
+ System.arraycopy(scratchDataPackedValue, 0, minPackedValue, 0, packedIndexBytesLength);
+ byte[] maxPackedValue = scratchMaxIndexPackedValue;
//Copy common prefixes before reading adjusted
// box
- System.arraycopy(minPackedValue, 0, maxPackedValue, 0, packedBytesLength);
+ System.arraycopy(minPackedValue, 0, maxPackedValue, 0, packedIndexBytesLength);
readMinMax(commonPrefixLengths, minPackedValue, maxPackedValue, in);
// The index gives us range of values for each dimension, but the actual range of values
@@ -467,24 +480,24 @@ public final class BKDReader extends PointValues implements Accountable {
int compressedDim = readCompressedDim(in);
if (compressedDim == -1) {
- visitRawDocValues(commonPrefixLengths, scratchPackedValue1, in, docIDs, count, visitor);
+ visitRawDocValues(commonPrefixLengths, scratchDataPackedValue, in, docIDs, count, visitor);
} else {
- visitCompressedDocValues(commonPrefixLengths, scratchPackedValue1, in, docIDs, count, visitor, compressedDim);
+ visitCompressedDocValues(commonPrefixLengths, scratchDataPackedValue, in, docIDs, count, visitor, compressedDim);
}
}
- private void readMinMax(int[] commonPrefixLengths, byte[] minPackedValue, byte[] maxPackedValue, IndexInput in) throws IOException {
- for (int dim = 0; dim < numDims; dim++) {
- int prefix = commonPrefixLengths[dim];
- in.readBytes(minPackedValue, dim * bytesPerDim + prefix, bytesPerDim - prefix);
- in.readBytes(maxPackedValue, dim * bytesPerDim + prefix, bytesPerDim - prefix);
- }
+ private void readMinMax(int[] commonPrefixLengths, byte[] minPackedValue, byte[] maxPackedValue, IndexInput in) throws IOException {
+ for (int dim = 0; dim < numIndexDims; dim++) {
+ int prefix = commonPrefixLengths[dim];
+ in.readBytes(minPackedValue, dim * bytesPerDim + prefix, bytesPerDim - prefix);
+ in.readBytes(maxPackedValue, dim * bytesPerDim + prefix, bytesPerDim - prefix);
}
+ }
// Just read suffixes for every dimension
private void visitRawDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException {
for (int i = 0; i < count; ++i) {
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
int prefix = commonPrefixLengths[dim];
in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix);
}
@@ -502,7 +515,7 @@ public final class BKDReader extends PointValues implements Accountable {
scratchPackedValue[compressedByteOffset] = in.readByte();
final int runLen = Byte.toUnsignedInt(in.readByte());
for (int j = 0; j < runLen; ++j) {
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
int prefix = commonPrefixLengths[dim];
in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix);
}
@@ -517,14 +530,14 @@ public final class BKDReader extends PointValues implements Accountable {
private int readCompressedDim(IndexInput in) throws IOException {
int compressedDim = in.readByte();
- if (compressedDim < -1 || compressedDim >= numDims) {
+ if (compressedDim < -1 || compressedDim >= numDataDims) {
throw new CorruptIndexException("Got compressedDim="+compressedDim, in);
}
return compressedDim;
}
private void readCommonPrefixes(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in) throws IOException {
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
int prefix = in.readVInt();
commonPrefixLengths[dim] = prefix;
if (prefix > 0) {
@@ -561,15 +574,15 @@ public final class BKDReader extends PointValues implements Accountable {
int count = readDocIDs(state.in, state.index.getLeafBlockFP(), state.scratchDocIDs);
// Again, this time reading values and checking with the visitor
- visitDocValues(state.commonPrefixLengths, state.scratchPackedValue1, state.scratchPackedValue2, state.in, state.scratchDocIDs, count, state.visitor);
+ visitDocValues(state.commonPrefixLengths, state.scratchDataPackedValue, state.scratchMinIndexPackedValue, state.scratchMaxIndexPackedValue, state.in, state.scratchDocIDs, count, state.visitor);
}
} else {
// Non-leaf node: recurse on the split left and right nodes
int splitDim = state.index.getSplitDim();
- assert splitDim >= 0: "splitDim=" + splitDim;
- assert splitDim < numDims;
+ assert splitDim >= 0: "splitDim=" + splitDim + ", numIndexDims=" + numIndexDims;
+ assert splitDim < numIndexDims: "splitDim=" + splitDim + ", numIndexDims=" + numIndexDims;
byte[] splitPackedValue = state.index.getSplitPackedValue();
BytesRef splitDimValue = state.index.getSplitDimValue();
@@ -577,11 +590,11 @@ public final class BKDReader extends PointValues implements Accountable {
//System.out.println(" splitDimValue=" + splitDimValue + " splitDim=" + splitDim);
// make sure cellMin <= splitValue <= cellMax:
- assert FutureArrays.compareUnsigned(cellMinPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) <= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numDims=" + numDims;
- assert FutureArrays.compareUnsigned(cellMaxPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) >= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numDims=" + numDims;
+ assert FutureArrays.compareUnsigned(cellMinPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) <= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numIndexDims=" + numIndexDims + " numDataDims=" + numDataDims;
+ assert FutureArrays.compareUnsigned(cellMaxPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) >= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numIndexDims=" + numIndexDims + " numDataDims=" + numDataDims;
// Recurse on left sub-tree:
- System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedBytesLength);
+ System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitDimValue.bytes, splitDimValue.offset, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
state.index.pushLeft();
intersect(state, cellMinPacked, splitPackedValue);
@@ -591,7 +604,7 @@ public final class BKDReader extends PointValues implements Accountable {
System.arraycopy(splitPackedValue, splitDim*bytesPerDim, splitDimValue.bytes, splitDimValue.offset, bytesPerDim);
// Recurse on right sub-tree:
- System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedBytesLength);
+ System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitDimValue.bytes, splitDimValue.offset, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
state.index.pushRight();
intersect(state, splitPackedValue, cellMaxPacked);
@@ -622,8 +635,8 @@ public final class BKDReader extends PointValues implements Accountable {
// Non-leaf node: recurse on the split left and right nodes
int splitDim = state.index.getSplitDim();
- assert splitDim >= 0: "splitDim=" + splitDim;
- assert splitDim < numDims;
+ assert splitDim >= 0: "splitDim=" + splitDim + ", numIndexDims=" + numIndexDims;
+ assert splitDim < numIndexDims: "splitDim=" + splitDim + ", numIndexDims=" + numIndexDims;
byte[] splitPackedValue = state.index.getSplitPackedValue();
BytesRef splitDimValue = state.index.getSplitDimValue();
@@ -631,11 +644,11 @@ public final class BKDReader extends PointValues implements Accountable {
//System.out.println(" splitDimValue=" + splitDimValue + " splitDim=" + splitDim);
// make sure cellMin <= splitValue <= cellMax:
- assert FutureArrays.compareUnsigned(cellMinPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) <= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numDims=" + numDims;
- assert FutureArrays.compareUnsigned(cellMaxPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) >= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numDims=" + numDims;
+ assert FutureArrays.compareUnsigned(cellMinPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) <= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numIndexDims=" + numIndexDims + " numDataDims=" + numDataDims;
+ assert FutureArrays.compareUnsigned(cellMaxPacked, splitDim * bytesPerDim, splitDim * bytesPerDim + bytesPerDim, splitDimValue.bytes, splitDimValue.offset, splitDimValue.offset + bytesPerDim) >= 0: "bytesPerDim=" + bytesPerDim + " splitDim=" + splitDim + " numIndexDims=" + numIndexDims + " numDataDims=" + numDataDims;
// Recurse on left sub-tree:
- System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedBytesLength);
+ System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitDimValue.bytes, splitDimValue.offset, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
state.index.pushLeft();
final long leftCost = estimatePointCount(state, cellMinPacked, splitPackedValue);
@@ -645,7 +658,7 @@ public final class BKDReader extends PointValues implements Accountable {
System.arraycopy(splitPackedValue, splitDim*bytesPerDim, splitDimValue.bytes, splitDimValue.offset, bytesPerDim);
// Recurse on right sub-tree:
- System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedBytesLength);
+ System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitDimValue.bytes, splitDimValue.offset, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
state.index.pushRight();
final long rightCost = estimatePointCount(state, splitPackedValue, cellMaxPacked);
@@ -670,8 +683,13 @@ public final class BKDReader extends PointValues implements Accountable {
}
@Override
- public int getNumDimensions() {
- return numDims;
+ public int getNumDataDimensions() {
+ return numDataDims;
+ }
+
+ @Override
+ public int getNumIndexDimensions() {
+ return numIndexDims;
}
@Override
[2/4] lucene-solr:master: LUCENE-8496: Selective indexing - modify
BKDReader/BKDWriter to allow users to select a fewer number of dimensions to
be used for creating the index than the total number of dimensions used for
field encoding. i.e., dimensions 0
Posted by nk...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 014d470..d7db94b 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -86,7 +86,8 @@ public class BKDWriter implements Closeable {
public static final int VERSION_START = 4; // version used by Lucene 7.0
//public static final int VERSION_CURRENT = VERSION_START;
public static final int VERSION_LEAF_STORES_BOUNDS = 5;
- public static final int VERSION_CURRENT = VERSION_LEAF_STORES_BOUNDS;
+ public static final int VERSION_SELECTIVE_INDEXING = 6;
+ public static final int VERSION_CURRENT = VERSION_SELECTIVE_INDEXING;
/** How many bytes each docs takes in the fixed-width offline format */
private final int bytesPerDoc;
@@ -100,15 +101,21 @@ public class BKDWriter implements Closeable {
/** Maximum number of dimensions */
public static final int MAX_DIMS = 8;
- /** How many dimensions we are indexing */
- protected final int numDims;
+ /** How many dimensions we are storing at the leaf (data) nodes */
+ protected final int numDataDims;
+
+ /** How many dimensions we are indexing in the internal nodes */
+ protected final int numIndexDims;
/** How many bytes each value in each dimension takes. */
protected final int bytesPerDim;
- /** numDims * bytesPerDim */
+ /** numDataDims * bytesPerDim */
protected final int packedBytesLength;
+ /** numIndexDims * bytesPerDim */
+ protected final int packedIndexBytesLength;
+
final TrackingDirectoryWrapper tempDir;
final String tempFileNamePrefix;
final double maxMBSortInHeap;
@@ -154,37 +161,39 @@ public class BKDWriter implements Closeable {
private final int maxDoc;
- public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim,
+ public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDataDims, int numIndexDims, int bytesPerDim,
int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount, boolean singleValuePerDoc) throws IOException {
- this(maxDoc, tempDir, tempFileNamePrefix, numDims, bytesPerDim, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount, singleValuePerDoc,
+ this(maxDoc, tempDir, tempFileNamePrefix, numDataDims, numIndexDims, bytesPerDim, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount, singleValuePerDoc,
totalPointCount > Integer.MAX_VALUE, Math.max(1, (long) maxMBSortInHeap), OfflineSorter.MAX_TEMPFILES);
}
- protected BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim,
+ protected BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDataDims, int numIndexDims, int bytesPerDim,
int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount,
boolean singleValuePerDoc, boolean longOrds, long offlineSorterBufferMB, int offlineSorterMaxTempFiles) throws IOException {
- verifyParams(numDims, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount);
+ verifyParams(numDataDims, numIndexDims, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount);
// We use tracking dir to deal with removing files on exception, so each place that
// creates temp files doesn't need crazy try/finally/sucess logic:
this.tempDir = new TrackingDirectoryWrapper(tempDir);
this.tempFileNamePrefix = tempFileNamePrefix;
this.maxPointsInLeafNode = maxPointsInLeafNode;
- this.numDims = numDims;
+ this.numDataDims = numDataDims;
+ this.numIndexDims = numIndexDims;
this.bytesPerDim = bytesPerDim;
this.totalPointCount = totalPointCount;
this.maxDoc = maxDoc;
this.offlineSorterBufferMB = OfflineSorter.BufferSize.megabytes(offlineSorterBufferMB);
this.offlineSorterMaxTempFiles = offlineSorterMaxTempFiles;
docsSeen = new FixedBitSet(maxDoc);
- packedBytesLength = numDims * bytesPerDim;
+ packedBytesLength = numDataDims * bytesPerDim;
+ packedIndexBytesLength = numIndexDims * bytesPerDim;
scratchDiff = new byte[bytesPerDim];
scratch1 = new byte[packedBytesLength];
scratch2 = new byte[packedBytesLength];
- commonPrefixLengths = new int[numDims];
+ commonPrefixLengths = new int[numDataDims];
- minPackedValue = new byte[packedBytesLength];
- maxPackedValue = new byte[packedBytesLength];
+ minPackedValue = new byte[packedIndexBytesLength];
+ maxPackedValue = new byte[packedIndexBytesLength];
// If we may have more than 1+Integer.MAX_VALUE values, then we must encode ords with long (8 bytes), else we can use int (4 bytes).
this.longOrds = longOrds;
@@ -211,7 +220,7 @@ public class BKDWriter implements Closeable {
// bytes to points here. Each dimension has its own sorted partition, so
// we must divide by numDims as wel.
- maxPointsSortInHeap = (int) (0.5 * (maxMBSortInHeap * 1024 * 1024) / (bytesPerDoc * numDims));
+ maxPointsSortInHeap = (int) (0.5 * (maxMBSortInHeap * 1024 * 1024) / (bytesPerDoc * numDataDims));
// Finally, we must be able to hold at least the leaf node in heap during build:
if (maxPointsSortInHeap < maxPointsInLeafNode) {
@@ -224,11 +233,14 @@ public class BKDWriter implements Closeable {
this.maxMBSortInHeap = maxMBSortInHeap;
}
- public static void verifyParams(int numDims, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount) {
+ public static void verifyParams(int numDataDims, int numIndexDims, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount) {
// We encode dim in a single byte in the splitPackedValues, but we only expose 4 bits for it now, in case we want to use
// remaining 4 bits for another purpose later
- if (numDims < 1 || numDims > MAX_DIMS) {
- throw new IllegalArgumentException("numDims must be 1 .. " + MAX_DIMS + " (got: " + numDims + ")");
+ if (numDataDims < 1 || numDataDims > MAX_DIMS) {
+ throw new IllegalArgumentException("numDataDims must be 1 .. " + MAX_DIMS + " (got: " + numDataDims + ")");
+ }
+ if (numIndexDims < 1 || numIndexDims > numDataDims) {
+ throw new IllegalArgumentException("numIndexDims must be 1 .. " + numDataDims + " (got: " + numIndexDims + ")");
}
if (maxPointsInLeafNode <= 0) {
throw new IllegalArgumentException("maxPointsInLeafNode must be > 0; got " + maxPointsInLeafNode);
@@ -277,10 +289,10 @@ public class BKDWriter implements Closeable {
// TODO: we could specialize for the 1D case:
if (pointCount == 0) {
- System.arraycopy(packedValue, 0, minPackedValue, 0, packedBytesLength);
- System.arraycopy(packedValue, 0, maxPackedValue, 0, packedBytesLength);
+ System.arraycopy(packedValue, 0, minPackedValue, 0, packedIndexBytesLength);
+ System.arraycopy(packedValue, 0, maxPackedValue, 0, packedIndexBytesLength);
} else {
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
int offset = dim*bytesPerDim;
if (FutureArrays.compareUnsigned(packedValue, offset, offset + bytesPerDim, minPackedValue, offset, offset + bytesPerDim) < 0) {
System.arraycopy(packedValue, offset, minPackedValue, offset, bytesPerDim);
@@ -325,8 +337,9 @@ public class BKDWriter implements Closeable {
public MergeReader(BKDReader bkd, MergeState.DocMap docMap) throws IOException {
this.bkd = bkd;
state = new BKDReader.IntersectState(bkd.in.clone(),
- bkd.numDims,
+ bkd.numDataDims,
bkd.packedBytesLength,
+ bkd.packedIndexBytesLength,
bkd.maxPointsInLeafNode,
null,
null);
@@ -347,7 +360,7 @@ public class BKDWriter implements Closeable {
docsInBlock = bkd.readDocIDs(state.in, state.in.getFilePointer(), state.scratchDocIDs);
assert docsInBlock > 0;
docBlockUpto = 0;
- bkd.visitDocValues(state.commonPrefixLengths, state.scratchPackedValue1, state.scratchPackedValue2, state.in, state.scratchDocIDs, docsInBlock, new IntersectVisitor() {
+ bkd.visitDocValues(state.commonPrefixLengths, state.scratchDataPackedValue, state.scratchMinIndexPackedValue, state.scratchMaxIndexPackedValue, state.in, state.scratchDocIDs, docsInBlock, new IntersectVisitor() {
int i = 0;
@Override
@@ -385,7 +398,7 @@ public class BKDWriter implements Closeable {
if (mappedDocID != -1) {
// Not deleted!
docID = mappedDocID;
- System.arraycopy(packedValues, index * bkd.packedBytesLength, state.scratchPackedValue1, 0, bkd.packedBytesLength);
+ System.arraycopy(packedValues, index * bkd.packedBytesLength, state.scratchDataPackedValue, 0, bkd.packedBytesLength);
return true;
}
}
@@ -404,7 +417,7 @@ public class BKDWriter implements Closeable {
public boolean lessThan(MergeReader a, MergeReader b) {
assert a != b;
- int cmp = FutureArrays.compareUnsigned(a.state.scratchPackedValue1, 0, bytesPerDim, b.state.scratchPackedValue1, 0, bytesPerDim);
+ int cmp = FutureArrays.compareUnsigned(a.state.scratchDataPackedValue, 0, bytesPerDim, b.state.scratchDataPackedValue, 0, bytesPerDim);
if (cmp < 0) {
return true;
} else if (cmp > 0) {
@@ -422,7 +435,7 @@ public class BKDWriter implements Closeable {
* disk. This method does not use transient disk in order to reorder points.
*/
public long writeField(IndexOutput out, String fieldName, MutablePointValues reader) throws IOException {
- if (numDims == 1) {
+ if (numIndexDims == 1) {
return writeField1Dim(out, fieldName, reader);
} else {
return writeFieldNDims(out, fieldName, reader);
@@ -465,7 +478,7 @@ public class BKDWriter implements Closeable {
Arrays.fill(maxPackedValue, (byte) 0);
for (int i = 0; i < Math.toIntExact(pointCount); ++i) {
values.getValue(i, scratchBytesRef1);
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
int offset = dim*bytesPerDim;
if (FutureArrays.compareUnsigned(scratchBytesRef1.bytes, scratchBytesRef1.offset + offset, scratchBytesRef1.offset + offset + bytesPerDim, minPackedValue, offset, offset + bytesPerDim) < 0) {
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + offset, minPackedValue, offset, bytesPerDim);
@@ -478,12 +491,12 @@ public class BKDWriter implements Closeable {
docsSeen.set(values.getDocID(i));
}
- final int[] parentSplits = new int[numDims];
+ final int[] parentSplits = new int[numIndexDims];
build(1, numLeaves, values, 0, Math.toIntExact(pointCount), out,
minPackedValue, maxPackedValue, parentSplits,
splitPackedValues, leafBlockFPs,
new int[maxPointsInLeafNode]);
- assert Arrays.equals(parentSplits, new int[numDims]);
+ assert Arrays.equals(parentSplits, new int[numIndexDims]);
long indexFP = out.getFilePointer();
writeIndex(out, Math.toIntExact(countPerLeaf), leafBlockFPs, splitPackedValues);
@@ -493,7 +506,7 @@ public class BKDWriter implements Closeable {
/* In the 1D case, we can simply sort points in ascending order and use the
* same writing logic as we use at merge time. */
private long writeField1Dim(IndexOutput out, String fieldName, MutablePointValues reader) throws IOException {
- MutablePointsReaderUtils.sort(maxDoc, packedBytesLength, reader, 0, Math.toIntExact(reader.size()));
+ MutablePointsReaderUtils.sort(maxDoc, packedIndexBytesLength, reader, 0, Math.toIntExact(reader.size()));
final OneDimensionBKDWriter oneDimWriter = new OneDimensionBKDWriter(out);
@@ -546,7 +559,7 @@ public class BKDWriter implements Closeable {
MergeReader reader = queue.top();
// System.out.println("iter reader=" + reader);
- oneDimWriter.add(reader.state.scratchPackedValue1, reader.docID);
+ oneDimWriter.add(reader.state.scratchDataPackedValue, reader.docID);
if (reader.next()) {
queue.updateTop();
@@ -573,8 +586,8 @@ public class BKDWriter implements Closeable {
private int leafCount;
OneDimensionBKDWriter(IndexOutput out) {
- if (numDims != 1) {
- throw new UnsupportedOperationException("numDims must be 1 but got " + numDims);
+ if (numIndexDims != 1) {
+ throw new UnsupportedOperationException("numIndexDims must be 1 but got " + numIndexDims);
}
if (pointCount != 0) {
throw new IllegalStateException("cannot mix add and merge");
@@ -651,9 +664,9 @@ public class BKDWriter implements Closeable {
private void writeLeafBlock() throws IOException {
assert leafCount != 0;
if (valueCount == 0) {
- System.arraycopy(leafValues, 0, minPackedValue, 0, packedBytesLength);
+ System.arraycopy(leafValues, 0, minPackedValue, 0, packedIndexBytesLength);
}
- System.arraycopy(leafValues, (leafCount - 1) * packedBytesLength, maxPackedValue, 0, packedBytesLength);
+ System.arraycopy(leafValues, (leafCount - 1) * packedBytesLength, maxPackedValue, 0, packedIndexBytesLength);
valueCount += leafCount;
@@ -828,7 +841,7 @@ public class BKDWriter implements Closeable {
*/
private PointWriter sort(int dim) throws IOException {
- assert dim >= 0 && dim < numDims;
+ assert dim >= 0 && dim < numDataDims;
if (heapPointWriter != null) {
@@ -861,7 +874,7 @@ public class BKDWriter implements Closeable {
final int offset = bytesPerDim * dim;
Comparator<BytesRef> cmp;
- if (dim == numDims - 1) {
+ if (dim == numDataDims - 1) {
// in that case the bytes for the dimension and for the doc id are contiguous,
// so we don't need a branch
cmp = new BytesRefComparator(bytesPerDim + Integer.BYTES) {
@@ -946,7 +959,7 @@ public class BKDWriter implements Closeable {
}
LongBitSet ordBitSet;
- if (numDims > 1) {
+ if (numDataDims > 1) {
if (singleValuePerDoc) {
ordBitSet = new LongBitSet(maxDoc);
} else {
@@ -981,7 +994,7 @@ public class BKDWriter implements Closeable {
assert pointCount / numLeaves <= maxPointsInLeafNode: "pointCount=" + pointCount + " numLeaves=" + numLeaves + " maxPointsInLeafNode=" + maxPointsInLeafNode;
// Sort all docs once by each dimension:
- PathSlice[] sortedPointWriters = new PathSlice[numDims];
+ PathSlice[] sortedPointWriters = new PathSlice[numDataDims];
// This is only used on exception; on normal code paths we close all files we opened:
List<Closeable> toCloseHeroically = new ArrayList<>();
@@ -989,7 +1002,9 @@ public class BKDWriter implements Closeable {
boolean success = false;
try {
//long t0 = System.nanoTime();
- for(int dim=0;dim<numDims;dim++) {
+ // even with selective indexing we create the sortedPointWriters so we can compress
+ // the leaf node data by common prefix
+ for(int dim=0;dim<numDataDims;dim++) {
sortedPointWriters[dim] = new PathSlice(sort(dim), 0, pointCount);
}
//long t1 = System.nanoTime();
@@ -1003,7 +1018,7 @@ public class BKDWriter implements Closeable {
heapPointWriter = null;
}
- final int[] parentSplits = new int[numDims];
+ final int[] parentSplits = new int[numIndexDims];
build(1, numLeaves, sortedPointWriters,
ordBitSet, out,
minPackedValue, maxPackedValue,
@@ -1011,7 +1026,7 @@ public class BKDWriter implements Closeable {
splitPackedValues,
leafBlockFPs,
toCloseHeroically);
- assert Arrays.equals(parentSplits, new int[numDims]);
+ assert Arrays.equals(parentSplits, new int[numIndexDims]);
for(PathSlice slice : sortedPointWriters) {
slice.writer.destroy();
@@ -1046,7 +1061,7 @@ public class BKDWriter implements Closeable {
// Possibly rotate the leaf block FPs, if the index not fully balanced binary tree (only happens
// if it was created by OneDimensionBKDWriter). In this case the leaf nodes may straddle the two bottom
// levels of the binary tree:
- if (numDims == 1 && numLeaves > 1) {
+ if (numIndexDims == 1 && numLeaves > 1) {
int levelCount = 2;
while (true) {
if (numLeaves >= levelCount && numLeaves <= 2*levelCount) {
@@ -1072,9 +1087,9 @@ public class BKDWriter implements Closeable {
// This is the "file" we append the byte[] to:
List<byte[]> blocks = new ArrayList<>();
- byte[] lastSplitValues = new byte[bytesPerDim * numDims];
+ byte[] lastSplitValues = new byte[bytesPerDim * numIndexDims];
//System.out.println("\npack index");
- int totalSize = recursePackIndex(writeBuffer, leafBlockFPs, splitPackedValues, 0l, blocks, 1, lastSplitValues, new boolean[numDims], false);
+ int totalSize = recursePackIndex(writeBuffer, leafBlockFPs, splitPackedValues, 0l, blocks, 1, lastSplitValues, new boolean[numIndexDims], false);
// Compact the byte[] blocks into single byte index:
byte[] index = new byte[totalSize];
@@ -1126,7 +1141,7 @@ public class BKDWriter implements Closeable {
if (isLeft == false) {
leftBlockFP = getLeftMostLeafBlockFP(leafBlockFPs, nodeID);
long delta = leftBlockFP - minBlockFP;
- assert nodeID == 1 || delta > 0;
+ assert nodeID == 1 || delta > 0 : "expected nodeID=1 or delta > 0; got nodeID=" + nodeID + " and delta=" + delta;
writeBuffer.writeVLong(delta);
} else {
// The left tree's left most leaf block FP is always the minimal FP:
@@ -1162,7 +1177,7 @@ public class BKDWriter implements Closeable {
}
// pack the prefix, splitDim and delta first diff byte into a single vInt:
- int code = (firstDiffByteDelta * (1+bytesPerDim) + prefix) * numDims + splitDim;
+ int code = (firstDiffByteDelta * (1+bytesPerDim) + prefix) * numIndexDims + splitDim;
//System.out.println(" code=" + code);
//System.out.println(" splitValue=" + new BytesRef(splitPackedValues, address, bytesPerDim));
@@ -1248,14 +1263,15 @@ public class BKDWriter implements Closeable {
private void writeIndex(IndexOutput out, int countPerLeaf, int numLeaves, byte[] packedIndex) throws IOException {
CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
- out.writeVInt(numDims);
+ out.writeVInt(numDataDims);
+ out.writeVInt(numIndexDims);
out.writeVInt(countPerLeaf);
out.writeVInt(bytesPerDim);
assert numLeaves > 0;
out.writeVInt(numLeaves);
- out.writeBytes(minPackedValue, 0, packedBytesLength);
- out.writeBytes(maxPackedValue, 0, packedBytesLength);
+ out.writeBytes(minPackedValue, 0, packedIndexBytesLength);
+ out.writeBytes(maxPackedValue, 0, packedIndexBytesLength);
out.writeVLong(pointCount);
out.writeVInt(docsSeen.cardinality());
@@ -1275,7 +1291,7 @@ public class BKDWriter implements Closeable {
// all values in this block are equal
out.writeByte((byte) -1);
} else {
- if (numDims != 1) {
+ if (numIndexDims != 1) {
writeActualBounds(out, commonPrefixLengths, count, packedValues);
}
assert commonPrefixLengths[sortedDim] < bytesPerDim;
@@ -1298,7 +1314,7 @@ public class BKDWriter implements Closeable {
}
private void writeActualBounds(DataOutput out, int[] commonPrefixLengths, int count, IntFunction<BytesRef> packedValues) throws IOException {
- for (int dim = 0; dim < numDims; ++dim) {
+ for (int dim = 0; dim < numIndexDims; ++dim) {
int commonPrefixLength = commonPrefixLengths[dim];
int suffixLength = bytesPerDim - commonPrefixLength;
if (suffixLength > 0) {
@@ -1336,7 +1352,7 @@ public class BKDWriter implements Closeable {
BytesRef ref = packedValues.apply(i);
assert ref.length == packedBytesLength;
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
int prefix = commonPrefixLengths[dim];
out.writeBytes(ref.bytes, ref.offset + dim*bytesPerDim + prefix, bytesPerDim-prefix);
}
@@ -1358,7 +1374,7 @@ public class BKDWriter implements Closeable {
}
private void writeCommonPrefixes(DataOutput out, int[] commonPrefixes, byte[] packedValue) throws IOException {
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
out.writeVInt(commonPrefixes[dim]);
//System.out.println(commonPrefixes[dim] + " of " + bytesPerDim);
out.writeBytes(packedValue, dim*bytesPerDim, commonPrefixes[dim]);
@@ -1429,7 +1445,7 @@ public class BKDWriter implements Closeable {
boolean result = reader.next();
assert result: "rightCount=" + rightCount + " source.count=" + source.count + " source.writer=" + source.writer;
System.arraycopy(reader.packedValue(), splitDim*bytesPerDim, scratch1, 0, bytesPerDim);
- if (numDims > 1) {
+ if (numDataDims > 1) {
assert ordBitSet.get(reader.ord()) == false;
ordBitSet.set(reader.ord());
// Subtract 1 from rightCount because we already did the first value above (so we could record the split value):
@@ -1444,7 +1460,7 @@ public class BKDWriter implements Closeable {
/** Called only in assert */
private boolean valueInBounds(BytesRef packedValue, byte[] minPackedValue, byte[] maxPackedValue) {
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
int offset = bytesPerDim*dim;
if (FutureArrays.compareUnsigned(packedValue.bytes, packedValue.offset + offset, packedValue.offset + offset + bytesPerDim, minPackedValue, offset, offset + bytesPerDim) < 0) {
return false;
@@ -1472,7 +1488,7 @@ public class BKDWriter implements Closeable {
for (int numSplits : parentSplits) {
maxNumSplits = Math.max(maxNumSplits, numSplits);
}
- for (int dim = 0; dim < numDims; ++dim) {
+ for (int dim = 0; dim < numIndexDims; ++dim) {
final int offset = dim * bytesPerDim;
if (parentSplits[dim] < maxNumSplits / 2 &&
FutureArrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, maxPackedValue, offset, offset + bytesPerDim) != 0) {
@@ -1482,7 +1498,7 @@ public class BKDWriter implements Closeable {
// Find which dim has the largest span so we can split on it:
int splitDim = -1;
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
NumericUtils.subtract(bytesPerDim, dim, maxPackedValue, minPackedValue, scratchDiff);
if (splitDim == -1 || FutureArrays.compareUnsigned(scratchDiff, 0, bytesPerDim, scratch1, 0, bytesPerDim) > 0) {
System.arraycopy(scratchDiff, 0, scratch1, 0, bytesPerDim);
@@ -1532,7 +1548,7 @@ public class BKDWriter implements Closeable {
reader.getValue(from, scratchBytesRef1);
for (int i = from + 1; i < to; ++i) {
reader.getValue(i, scratchBytesRef2);
- for (int dim=0;dim<numDims;dim++) {
+ for (int dim=0;dim<numDataDims;dim++) {
final int offset = dim * bytesPerDim;
for(int j=0;j<commonPrefixLengths[dim];j++) {
if (scratchBytesRef1.bytes[scratchBytesRef1.offset+offset+j] != scratchBytesRef2.bytes[scratchBytesRef2.offset+offset+j]) {
@@ -1544,14 +1560,14 @@ public class BKDWriter implements Closeable {
}
// Find the dimension that has the least number of unique bytes at commonPrefixLengths[dim]
- FixedBitSet[] usedBytes = new FixedBitSet[numDims];
- for (int dim = 0; dim < numDims; ++dim) {
+ FixedBitSet[] usedBytes = new FixedBitSet[numDataDims];
+ for (int dim = 0; dim < numDataDims; ++dim) {
if (commonPrefixLengths[dim] < bytesPerDim) {
usedBytes[dim] = new FixedBitSet(256);
}
}
for (int i = from + 1; i < to; ++i) {
- for (int dim=0;dim<numDims;dim++) {
+ for (int dim=0;dim<numDataDims;dim++) {
if (usedBytes[dim] != null) {
byte b = reader.getByteAt(i, dim * bytesPerDim + commonPrefixLengths[dim]);
usedBytes[dim].set(Byte.toUnsignedInt(b));
@@ -1560,7 +1576,7 @@ public class BKDWriter implements Closeable {
}
int sortedDim = 0;
int sortedDimCardinality = Integer.MAX_VALUE;
- for (int dim = 0; dim < numDims; ++dim) {
+ for (int dim = 0; dim < numDataDims; ++dim) {
if (usedBytes[dim] != null) {
final int cardinality = usedBytes[dim].cardinality();
if (cardinality < sortedDimCardinality) {
@@ -1631,8 +1647,8 @@ public class BKDWriter implements Closeable {
reader.getValue(mid, scratchBytesRef1);
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + splitDim * bytesPerDim, splitPackedValues, address + 1, bytesPerDim);
- byte[] minSplitPackedValue = ArrayUtil.copyOfSubArray(minPackedValue, 0, packedBytesLength);
- byte[] maxSplitPackedValue = ArrayUtil.copyOfSubArray(maxPackedValue, 0, packedBytesLength);
+ byte[] minSplitPackedValue = ArrayUtil.copyOfSubArray(minPackedValue, 0, packedIndexBytesLength);
+ byte[] maxSplitPackedValue = ArrayUtil.copyOfSubArray(maxPackedValue, 0, packedIndexBytesLength);
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + splitDim * bytesPerDim,
minSplitPackedValue, splitDim * bytesPerDim, bytesPerDim);
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + splitDim * bytesPerDim,
@@ -1666,7 +1682,7 @@ public class BKDWriter implements Closeable {
assert slice.count == slices[0].count;
}
- if (numDims == 1 && slices[0].writer instanceof OfflinePointWriter && slices[0].count <= maxPointsSortInHeap) {
+ if (numDataDims == 1 && slices[0].writer instanceof OfflinePointWriter && slices[0].count <= maxPointsSortInHeap) {
// Special case for 1D, to cutover to heap once we recurse deeply enough:
slices[0] = switchToHeap(slices[0], toCloseHeroically);
}
@@ -1679,7 +1695,7 @@ public class BKDWriter implements Closeable {
int sortedDim = 0;
int sortedDimCardinality = Integer.MAX_VALUE;
- for (int dim=0;dim<numDims;dim++) {
+ for (int dim=0;dim<numDataDims;dim++) {
if (slices[dim].writer instanceof HeapPointWriter == false) {
// Adversarial cases can cause this, e.g. very lopsided data, all equal points, such that we started
// offline, but then kept splitting only in one dimension, and so never had to rewrite into heap writer
@@ -1767,7 +1783,7 @@ public class BKDWriter implements Closeable {
// Inner node: partition/recurse
int splitDim;
- if (numDims > 1) {
+ if (numIndexDims > 1) {
splitDim = split(minPackedValue, maxPackedValue, parentSplits);
} else {
splitDim = 0;
@@ -1788,24 +1804,24 @@ public class BKDWriter implements Closeable {
// Partition all PathSlice that are not the split dim into sorted left and right sets, so we can recurse:
- PathSlice[] leftSlices = new PathSlice[numDims];
- PathSlice[] rightSlices = new PathSlice[numDims];
+ PathSlice[] leftSlices = new PathSlice[numDataDims];
+ PathSlice[] rightSlices = new PathSlice[numDataDims];
- byte[] minSplitPackedValue = new byte[packedBytesLength];
- System.arraycopy(minPackedValue, 0, minSplitPackedValue, 0, packedBytesLength);
+ byte[] minSplitPackedValue = new byte[packedIndexBytesLength];
+ System.arraycopy(minPackedValue, 0, minSplitPackedValue, 0, packedIndexBytesLength);
- byte[] maxSplitPackedValue = new byte[packedBytesLength];
- System.arraycopy(maxPackedValue, 0, maxSplitPackedValue, 0, packedBytesLength);
+ byte[] maxSplitPackedValue = new byte[packedIndexBytesLength];
+ System.arraycopy(maxPackedValue, 0, maxSplitPackedValue, 0, packedIndexBytesLength);
// When we are on this dim, below, we clear the ordBitSet:
int dimToClear;
- if (numDims - 1 == splitDim) {
- dimToClear = numDims - 2;
+ if (numDataDims - 1 == splitDim) {
+ dimToClear = numDataDims - 2;
} else {
- dimToClear = numDims - 1;
+ dimToClear = numDataDims - 1;
}
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
if (dim == splitDim) {
// No need to partition on this dim since it's a simple slice of the incoming already sorted slice, and we
@@ -1842,7 +1858,7 @@ public class BKDWriter implements Closeable {
ordBitSet, out,
minPackedValue, maxSplitPackedValue, parentSplits,
splitPackedValues, leafBlockFPs, toCloseHeroically);
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
// Don't destroy the dim we split on because we just re-used what our caller above gave us for that dim:
if (dim != splitDim) {
leftSlices[dim].writer.destroy();
@@ -1855,7 +1871,7 @@ public class BKDWriter implements Closeable {
ordBitSet, out,
minSplitPackedValue, maxPackedValue, parentSplits,
splitPackedValues, leafBlockFPs, toCloseHeroically);
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
// Don't destroy the dim we split on because we just re-used what our caller above gave us for that dim:
if (dim != splitDim) {
rightSlices[dim].writer.destroy();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java b/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java
index 9214cb9..b770e62 100644
--- a/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java
+++ b/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java
@@ -69,7 +69,7 @@ public class TestFieldType extends LuceneTestCase {
public void testPointsToString() {
FieldType ft = new FieldType();
ft.setDimensions(1, Integer.BYTES);
- assertEquals("pointDimensionCount=1,pointNumBytes=4", ft.toString());
+ assertEquals("pointDataDimensionCount=1,pointIndexDimensionCount=1,pointNumBytes=4", ft.toString());
}
private static Object randomValue(Class<?> clazz) {
@@ -85,15 +85,20 @@ public class TestFieldType extends LuceneTestCase {
private static FieldType randomFieldType() throws Exception {
// setDimensions handled special as values must be in-bounds.
- Method setDimensionsMethod = FieldType.class.getMethod("setDimensions", int.class, int.class);
+ Method setDimensionsMethodA = FieldType.class.getMethod("setDimensions", int.class, int.class);
+ Method setDimensionsMethodB = FieldType.class.getMethod("setDimensions", int.class, int.class, int.class);
FieldType ft = new FieldType();
for (Method method : FieldType.class.getMethods()) {
if (method.getName().startsWith("set")) {
final Class<?>[] parameterTypes = method.getParameterTypes();
final Object[] args = new Object[parameterTypes.length];
- if (method.equals(setDimensionsMethod)) {
+ if (method.equals(setDimensionsMethodA)) {
args[0] = 1 + random().nextInt(PointValues.MAX_DIMENSIONS);
args[1] = 1 + random().nextInt(PointValues.MAX_NUM_BYTES);
+ } else if (method.equals(setDimensionsMethodB)) {
+ args[0] = 1 + random().nextInt(PointValues.MAX_DIMENSIONS);
+ args[1] = 1 + random().nextInt((Integer)args[0]);
+ args[2] = 1 + random().nextInt(PointValues.MAX_NUM_BYTES);
} else {
for (int i = 0; i < args.length; ++i) {
args[i] = randomValue(parameterTypes[i]);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
index 25b48c8..1091b24 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
@@ -91,7 +91,12 @@ public class TestIndexableField extends LuceneTestCase {
}
@Override
- public int pointDimensionCount() {
+ public int pointDataDimensionCount() {
+ return 0;
+ }
+
+ @Override
+ public int pointIndexDimensionCount() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
index 7d03c7e..c882a3b 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
@@ -164,7 +164,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
deletes.onNewReader(segmentReader, commitInfo);
reader.close();
writer.close();
- FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 0, Collections.emptyMap(), 0, 0, true);
+ FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 0, Collections.emptyMap(), 0, 0, 0, true);
List<Integer> docsDeleted = Arrays.asList(1, 3, 7, 8, DocIdSetIterator.NO_MORE_DOCS);
List<DocValuesFieldUpdates> updates = Arrays.asList(singleUpdate(docsDeleted, 10, true));
for (DocValuesFieldUpdates update : updates) {
@@ -185,7 +185,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
docsDeleted = Arrays.asList(1, 2, DocIdSetIterator.NO_MORE_DOCS);
updates = Arrays.asList(singleUpdate(docsDeleted, 10, true));
- fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 1, Collections.emptyMap(), 0, 0, true);
+ fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 1, Collections.emptyMap(), 0, 0, 0, true);
for (DocValuesFieldUpdates update : updates) {
deletes.onDocValuesUpdate(fieldInfo, update.iterator());
}
@@ -228,7 +228,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
SegmentCommitInfo segmentInfo = segmentReader.getSegmentInfo();
PendingDeletes deletes = newPendingDeletes(segmentInfo);
deletes.onNewReader(segmentReader, segmentInfo);
- FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, true);
+ FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, 0, true);
List<Integer> docsDeleted = Arrays.asList(1, DocIdSetIterator.NO_MORE_DOCS);
List<DocValuesFieldUpdates> updates = Arrays.asList(singleUpdate(docsDeleted, 3, true));
for (DocValuesFieldUpdates update : updates) {
@@ -276,7 +276,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
SegmentCommitInfo segmentInfo = segmentReader.getSegmentInfo();
PendingDeletes deletes = newPendingDeletes(segmentInfo);
deletes.onNewReader(segmentReader, segmentInfo);
- FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, true);
+ FieldInfo fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, 0, true);
List<DocValuesFieldUpdates> updates = Arrays.asList(singleUpdate(Arrays.asList(0, 1, DocIdSetIterator.NO_MORE_DOCS), 3, false));
for (DocValuesFieldUpdates update : updates) {
deletes.onDocValuesUpdate(fieldInfo, update.iterator());
@@ -295,7 +295,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
assertEquals(0, deletes.numPendingDeletes());
segmentInfo.advanceDocValuesGen();
- fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, true);
+ fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, segmentInfo.getNextDocValuesGen(), Collections.emptyMap(), 0, 0, 0, true);
updates = Arrays.asList(singleUpdate(Arrays.asList(1, DocIdSetIterator.NO_MORE_DOCS), 3, true));
for (DocValuesFieldUpdates update : updates) {
deletes.onDocValuesUpdate(fieldInfo, update.iterator());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
index 1ef7abc..ae8605d 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
@@ -73,7 +73,7 @@ public class TestPointValues extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
w.addDocument(doc);
});
- assertEquals("cannot change point dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
+ assertEquals("cannot change point data dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
w.close();
dir.close();
}
@@ -91,7 +91,7 @@ public class TestPointValues extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
w.addDocument(doc2);
});
- assertEquals("cannot change point dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
+ assertEquals("cannot change point data dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
w.close();
dir.close();
@@ -111,7 +111,7 @@ public class TestPointValues extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
w.addDocument(doc2);
});
- assertEquals("cannot change point dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
+ assertEquals("cannot change point data dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
w.close();
dir.close();
@@ -133,7 +133,7 @@ public class TestPointValues extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
w2.addDocument(doc2);
});
- assertEquals("cannot change point dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
+ assertEquals("cannot change point data dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
w2.close();
dir.close();
@@ -156,7 +156,7 @@ public class TestPointValues extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
w2.addIndexes(new Directory[] {dir});
});
- assertEquals("cannot change point dimension count from 2 to 1 for field=\"dim\"", expected.getMessage());
+ assertEquals("cannot change point data dimension count from 2 to 1 for field=\"dim\"", expected.getMessage());
IOUtils.close(w2, dir, dir2);
}
@@ -179,7 +179,7 @@ public class TestPointValues extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
w2.addIndexes(new CodecReader[] {(CodecReader) getOnlyLeafReader(r)});
});
- assertEquals("cannot change point dimension count from 2 to 1 for field=\"dim\"", expected.getMessage());
+ assertEquals("cannot change point data dimension count from 2 to 1 for field=\"dim\"", expected.getMessage());
IOUtils.close(r, w2, dir, dir2);
}
@@ -203,7 +203,7 @@ public class TestPointValues extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
TestUtil.addIndexesSlowly(w2, r);
});
- assertEquals("cannot change point dimension count from 2 to 1 for field=\"dim\"", expected.getMessage());
+ assertEquals("cannot change point data dimension count from 2 to 1 for field=\"dim\"", expected.getMessage());
IOUtils.close(r, w2, dir, dir2);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
index f76b973..90df7c3 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
@@ -1095,7 +1095,7 @@ public class TestPointQueries extends LuceneTestCase {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
s.count(BinaryPoint.newRangeQuery("value", point, point));
});
- assertEquals("field=\"value\" was indexed with numDims=1 but this query has numDims=2", expected.getMessage());
+ assertEquals("field=\"value\" was indexed with numIndexDimensions=1 but this query has numDims=2", expected.getMessage());
IOUtils.close(r, w, dir);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java b/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java
index f87a73a..a51b161 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java
@@ -327,7 +327,12 @@ public class TestDocIdSetBuilder extends LuceneTestCase {
}
@Override
- public int getNumDimensions() throws IOException {
+ public int getNumDataDimensions() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getNumIndexDimensions() throws IOException {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
index e30168c..0d57bf8 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java
@@ -41,7 +41,7 @@ public class Test2BBKDPoints extends LuceneTestCase {
final int numDocs = (Integer.MAX_VALUE / 26) + 100;
- BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, Long.BYTES,
+ BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, 1, Long.BYTES,
BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, 26L * numDocs, false);
int counter = 0;
byte[] packedBytes = new byte[Long.BYTES];
@@ -78,7 +78,7 @@ public class Test2BBKDPoints extends LuceneTestCase {
final int numDocs = (Integer.MAX_VALUE / 26) + 100;
- BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES,
+ BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, 2, Long.BYTES,
BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, 26L * numDocs, false);
int counter = 0;
byte[] packedBytes = new byte[2*Long.BYTES];
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
index cbd4d37..d75d785 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
@@ -48,7 +48,7 @@ public class TestBKD extends LuceneTestCase {
public void testBasicInts1D() throws Exception {
try (Directory dir = getDirectory(100)) {
- BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 4, 2, 1.0f, 100, true);
+ BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 1, 4, 2, 1.0f, 100, true);
byte[] scratch = new byte[4];
for(int docID=0;docID<100;docID++) {
NumericUtils.intToSortableBytes(docID, scratch, 0);
@@ -121,12 +121,13 @@ public class TestBKD extends LuceneTestCase {
int numDocs = atLeast(1000);
try (Directory dir = getDirectory(numDocs)) {
int numDims = TestUtil.nextInt(random(), 1, 5);
+ int numIndexDims = TestUtil.nextInt(random(), 1, numDims);
int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100);
float maxMB = (float) 3.0 + (3*random().nextFloat());
- BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, 4, maxPointsInLeafNode, maxMB, numDocs, true);
+ BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numIndexDims, 4, maxPointsInLeafNode, maxMB, numDocs, true);
if (VERBOSE) {
- System.out.println("TEST: numDims=" + numDims + " numDocs=" + numDocs);
+ System.out.println("TEST: numDims=" + numDims + " numIndexDims=" + numIndexDims + " numDocs=" + numDocs);
}
int[][] docs = new int[numDocs][];
byte[] scratch = new byte[4*numDims];
@@ -167,7 +168,7 @@ public class TestBKD extends LuceneTestCase {
byte[] minPackedValue = r.getMinPackedValue();
byte[] maxPackedValue = r.getMaxPackedValue();
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
assertEquals(minValue[dim], NumericUtils.sortableBytesToInt(minPackedValue, dim * Integer.BYTES));
assertEquals(maxValue[dim], NumericUtils.sortableBytesToInt(maxPackedValue, dim * Integer.BYTES));
}
@@ -181,7 +182,7 @@ public class TestBKD extends LuceneTestCase {
// Random N dims rect query:
int[] queryMin = new int[numDims];
int[] queryMax = new int[numDims];
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
queryMin[dim] = random().nextInt();
queryMax[dim] = random().nextInt();
if (queryMin[dim] > queryMax[dim]) {
@@ -202,7 +203,7 @@ public class TestBKD extends LuceneTestCase {
@Override
public void visit(int docID, byte[] packedValue) {
//System.out.println("visit check docID=" + docID);
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
int x = NumericUtils.sortableBytesToInt(packedValue, dim * Integer.BYTES);
if (x < queryMin[dim] || x > queryMax[dim]) {
//System.out.println(" no");
@@ -217,7 +218,7 @@ public class TestBKD extends LuceneTestCase {
@Override
public Relation compare(byte[] minPacked, byte[] maxPacked) {
boolean crosses = false;
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
int min = NumericUtils.sortableBytesToInt(minPacked, dim * Integer.BYTES);
int max = NumericUtils.sortableBytesToInt(maxPacked, dim * Integer.BYTES);
assert max >= min;
@@ -240,7 +241,7 @@ public class TestBKD extends LuceneTestCase {
for(int docID=0;docID<numDocs;docID++) {
int[] docValues = docs[docID];
boolean expected = true;
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
int x = docValues[dim];
if (x < queryMin[dim] || x > queryMax[dim]) {
expected = false;
@@ -264,7 +265,7 @@ public class TestBKD extends LuceneTestCase {
int numDims = TestUtil.nextInt(random(), 1, 5);
int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100);
float maxMB = (float) 3.0 + (3*random().nextFloat());
- BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, numDocs, true);
+ BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, numDocs, true);
BigInteger[][] docs = new BigInteger[numDocs][];
byte[] scratch = new byte[numBytesPerDim*numDims];
@@ -380,13 +381,14 @@ public class TestBKD extends LuceneTestCase {
public void testWithExceptions() throws Exception {
int numDocs = atLeast(10000);
int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
- int numDims = TestUtil.nextInt(random(), 1, 5);
+ int numDataDims = TestUtil.nextInt(random(), 1, 5);
+ int numIndexDims = TestUtil.nextInt(random(), 1, numDataDims);
byte[][][] docValues = new byte[numDocs][][];
for(int docID=0;docID<numDocs;docID++) {
- byte[][] values = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
+ byte[][] values = new byte[numDataDims][];
+ for(int dim=0;dim<numDataDims;dim++) {
values[dim] = new byte[numBytesPerDim];
random().nextBytes(values[dim]);
}
@@ -401,7 +403,7 @@ public class TestBKD extends LuceneTestCase {
try {
dir.setRandomIOExceptionRate(0.05);
dir.setRandomIOExceptionRateOnOpen(0.05);
- verify(dir, docValues, null, numDims, numBytesPerDim, 50, maxMBHeap);
+ verify(dir, docValues, null, numDataDims, numIndexDims, numBytesPerDim, 50, maxMBHeap);
} catch (IllegalArgumentException iae) {
// This just means we got a too-small maxMB for the maxPointsInLeafNode; just retry w/ more heap
assertTrue(iae.getMessage().contains("either increase maxMBSortInHeap or decrease maxPointsInLeafNode"));
@@ -439,7 +441,7 @@ public class TestBKD extends LuceneTestCase {
public void testTooLittleHeap() throws Exception {
try (Directory dir = getDirectory(0)) {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
- new BKDWriter(1, dir, "bkd", 1, 16, 1000000, 0.001, 0, true);
+ new BKDWriter(1, dir, "bkd", 1, 1, 16, 1000000, 0.001, 0, true);
});
assertTrue(expected.getMessage().contains("either increase maxMBSortInHeap or decrease maxPointsInLeafNode"));
}
@@ -449,33 +451,35 @@ public class TestBKD extends LuceneTestCase {
int numDocs = TestUtil.nextInt(random(), count, count*2);
int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
- int numDims = TestUtil.nextInt(random(), 1, 5);
+ int numDataDims = TestUtil.nextInt(random(), 1, 5);
+ int numIndexDims = TestUtil.nextInt(random(), 1, numDataDims);
byte[][][] docValues = new byte[numDocs][][];
for(int docID=0;docID<numDocs;docID++) {
- byte[][] values = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
+ byte[][] values = new byte[numDataDims][];
+ for(int dim=0;dim<numDataDims;dim++) {
values[dim] = new byte[numBytesPerDim];
random().nextBytes(values[dim]);
}
docValues[docID] = values;
}
- verify(docValues, null, numDims, numBytesPerDim);
+ verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim);
}
public void testAllEqual() throws Exception {
int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
- int numDims = TestUtil.nextInt(random(), 1, 5);
+ int numDataDims = TestUtil.nextInt(random(), 1, 5);
+ int numIndexDims = TestUtil.nextInt(random(), 1, numDataDims);
int numDocs = atLeast(1000);
byte[][][] docValues = new byte[numDocs][][];
for(int docID=0;docID<numDocs;docID++) {
if (docID == 0) {
- byte[][] values = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
+ byte[][] values = new byte[numDataDims][];
+ for(int dim=0;dim<numDataDims;dim++) {
values[dim] = new byte[numBytesPerDim];
random().nextBytes(values[dim]);
}
@@ -485,20 +489,21 @@ public class TestBKD extends LuceneTestCase {
}
}
- verify(docValues, null, numDims, numBytesPerDim);
+ verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim);
}
public void testOneDimEqual() throws Exception {
int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
- int numDims = TestUtil.nextInt(random(), 1, 5);
+ int numDataDims = TestUtil.nextInt(random(), 1, 5);
+ int numIndexDims = TestUtil.nextInt(random(), 1, numDataDims);
int numDocs = atLeast(1000);
- int theEqualDim = random().nextInt(numDims);
+ int theEqualDim = random().nextInt(numDataDims);
byte[][][] docValues = new byte[numDocs][][];
for(int docID=0;docID<numDocs;docID++) {
- byte[][] values = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
+ byte[][] values = new byte[numDataDims][];
+ for(int dim=0;dim<numDataDims;dim++) {
values[dim] = new byte[numBytesPerDim];
random().nextBytes(values[dim]);
}
@@ -509,17 +514,18 @@ public class TestBKD extends LuceneTestCase {
}
// Use a small number of points in leaf blocks to trigger a lot of splitting
- verify(docValues, null, numDims, numBytesPerDim, TestUtil.nextInt(random(), 20, 50));
+ verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim, TestUtil.nextInt(random(), 20, 50));
}
// This triggers the logic that makes sure all dimensions get indexed
// by looking at how many times each dim has been split
public void testOneDimLowCard() throws Exception {
int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
- int numDims = TestUtil.nextInt(random(), 2, 5);
+ int numDataDims = TestUtil.nextInt(random(), 2, 5);
+ int numIndexDims = TestUtil.nextInt(random(), 2, numDataDims);
int numDocs = atLeast(10000);
- int theLowCardDim = random().nextInt(numDims);
+ int theLowCardDim = random().nextInt(numDataDims);
byte[] value1 = new byte[numBytesPerDim];
random().nextBytes(value1);
@@ -533,8 +539,8 @@ public class TestBKD extends LuceneTestCase {
byte[][][] docValues = new byte[numDocs][][];
for(int docID=0;docID<numDocs;docID++) {
- byte[][] values = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
+ byte[][] values = new byte[numDataDims][];
+ for(int dim=0;dim<numDataDims;dim++) {
if (dim == theLowCardDim) {
values[dim] = random().nextBoolean() ? value1 : value2;
} else {
@@ -546,16 +552,17 @@ public class TestBKD extends LuceneTestCase {
}
// Use a small number of points in leaf blocks to trigger a lot of splitting
- verify(docValues, null, numDims, numBytesPerDim, TestUtil.nextInt(random(), 20, 50));
+ verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim, TestUtil.nextInt(random(), 20, 50));
}
// this should trigger run-length compression with lengths that are greater than 255
public void testOneDimTwoValues() throws Exception {
int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
- int numDims = TestUtil.nextInt(random(), 1, 5);
+ int numDataDims = TestUtil.nextInt(random(), 1, 5);
+ int numIndexDims = TestUtil.nextInt(random(), 1, numDataDims);
int numDocs = atLeast(1000);
- int theDim = random().nextInt(numDims);
+ int theDim = random().nextInt(numDataDims);
byte[] value1 = new byte[numBytesPerDim];
random().nextBytes(value1);
byte[] value2 = new byte[numBytesPerDim];
@@ -563,8 +570,8 @@ public class TestBKD extends LuceneTestCase {
byte[][][] docValues = new byte[numDocs][][];
for(int docID=0;docID<numDocs;docID++) {
- byte[][] values = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
+ byte[][] values = new byte[numDataDims][];
+ for(int dim=0;dim<numDataDims;dim++) {
if (dim == theDim) {
values[dim] = random().nextBoolean() ? value1 : value2;
} else {
@@ -575,12 +582,13 @@ public class TestBKD extends LuceneTestCase {
docValues[docID] = values;
}
- verify(docValues, null, numDims, numBytesPerDim);
+ verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim);
}
public void testMultiValued() throws Exception {
int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
- int numDims = TestUtil.nextInt(random(), 1, 5);
+ int numDataDims = TestUtil.nextInt(random(), 1, 5);
+ int numIndexDims = TestUtil.nextInt(random(), 1, numDataDims);
int numDocs = atLeast(1000);
List<byte[][]> docValues = new ArrayList<>();
@@ -590,8 +598,8 @@ public class TestBKD extends LuceneTestCase {
int numValuesInDoc = TestUtil.nextInt(random(), 1, 5);
for(int ord=0;ord<numValuesInDoc;ord++) {
docIDs.add(docID);
- byte[][] values = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
+ byte[][] values = new byte[numDataDims][];
+ for(int dim=0;dim<numDataDims;dim++) {
values[dim] = new byte[numBytesPerDim];
random().nextBytes(values[dim]);
}
@@ -605,33 +613,33 @@ public class TestBKD extends LuceneTestCase {
docIDsArray[i] = docIDs.get(i);
}
- verify(docValuesArray, docIDsArray, numDims, numBytesPerDim);
+ verify(docValuesArray, docIDsArray, numDataDims, numIndexDims, numBytesPerDim);
}
/** docIDs can be null, for the single valued case, else it maps value to docID */
- private void verify(byte[][][] docValues, int[] docIDs, int numDims, int numBytesPerDim) throws Exception {
- verify(docValues, docIDs, numDims, numBytesPerDim, TestUtil.nextInt(random(), 50, 1000));
+ private void verify(byte[][][] docValues, int[] docIDs, int numDataDims, int numIndexDims, int numBytesPerDim) throws Exception {
+ verify(docValues, docIDs, numDataDims, numIndexDims, numBytesPerDim, TestUtil.nextInt(random(), 50, 1000));
}
- private void verify(byte[][][] docValues, int[] docIDs, int numDims, int numBytesPerDim,
+ private void verify(byte[][][] docValues, int[] docIDs, int numDataDims, int numIndexDims, int numBytesPerDim,
int maxPointsInLeafNode) throws Exception {
try (Directory dir = getDirectory(docValues.length)) {
double maxMB = (float) 3.0 + (3*random().nextDouble());
- verify(dir, docValues, docIDs, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
+ verify(dir, docValues, docIDs, numDataDims, numIndexDims, numBytesPerDim, maxPointsInLeafNode, maxMB);
}
}
- private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDims, int numBytesPerDim, int maxPointsInLeafNode, double maxMB) throws Exception {
+ private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDataDims, int numIndexDims, int numBytesPerDim, int maxPointsInLeafNode, double maxMB) throws Exception {
int numValues = docValues.length;
if (VERBOSE) {
- System.out.println("TEST: numValues=" + numValues + " numDims=" + numDims + " numBytesPerDim=" + numBytesPerDim + " maxPointsInLeafNode=" + maxPointsInLeafNode + " maxMB=" + maxMB);
+ System.out.println("TEST: numValues=" + numValues + " numDataDims=" + numDataDims + " numIndexDims=" + numIndexDims + " numBytesPerDim=" + numBytesPerDim + " maxPointsInLeafNode=" + maxPointsInLeafNode + " maxMB=" + maxMB);
}
List<Long> toMerge = null;
List<MergeState.DocMap> docMaps = null;
int seg = 0;
- BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
+ BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDataDims, numIndexDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT);
IndexInput in = null;
@@ -639,9 +647,9 @@ public class TestBKD extends LuceneTestCase {
try {
- byte[] scratch = new byte[numBytesPerDim*numDims];
+ byte[] scratch = new byte[numBytesPerDim*numDataDims];
int lastDocIDBase = 0;
- boolean useMerge = numDims == 1 && numValues >= 10 && random().nextBoolean();
+ boolean useMerge = numDataDims == 1 && numValues >= 10 && random().nextBoolean();
int valuesInThisSeg;
if (useMerge) {
// Sometimes we will call merge with a single segment:
@@ -662,7 +670,7 @@ public class TestBKD extends LuceneTestCase {
if (VERBOSE) {
System.out.println(" ord=" + ord + " docID=" + docID + " lastDocIDBase=" + lastDocIDBase);
}
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
if (VERBOSE) {
System.out.println(" " + dim + " -> " + new BytesRef(docValues[ord][dim]));
}
@@ -691,7 +699,7 @@ public class TestBKD extends LuceneTestCase {
seg++;
maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 1000);
maxMB = (float) 3.0 + (3*random().nextDouble());
- w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
+ w = new BKDWriter(numValues, dir, "_" + seg, numDataDims, numIndexDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
lastDocIDBase = docID;
}
}
@@ -712,7 +720,7 @@ public class TestBKD extends LuceneTestCase {
out.close();
in = dir.openInput("bkd", IOContext.DEFAULT);
seg++;
- w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
+ w = new BKDWriter(numValues, dir, "_" + seg, numDataDims, numIndexDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
List<BKDReader> readers = new ArrayList<>();
for(long fp : toMerge) {
in.seek(fp);
@@ -739,9 +747,9 @@ public class TestBKD extends LuceneTestCase {
}
// Random N dims rect query:
- byte[][] queryMin = new byte[numDims][];
- byte[][] queryMax = new byte[numDims][];
- for(int dim=0;dim<numDims;dim++) {
+ byte[][] queryMin = new byte[numDataDims][];
+ byte[][] queryMax = new byte[numDataDims][];
+ for(int dim=0;dim<numDataDims;dim++) {
queryMin[dim] = new byte[numBytesPerDim];
random().nextBytes(queryMin[dim]);
queryMax[dim] = new byte[numBytesPerDim];
@@ -764,7 +772,7 @@ public class TestBKD extends LuceneTestCase {
@Override
public void visit(int docID, byte[] packedValue) {
//System.out.println("visit check docID=" + docID);
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
if (FutureArrays.compareUnsigned(packedValue, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMin[dim], 0, numBytesPerDim) < 0 ||
FutureArrays.compareUnsigned(packedValue, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMax[dim], 0, numBytesPerDim) > 0) {
//System.out.println(" no");
@@ -779,7 +787,7 @@ public class TestBKD extends LuceneTestCase {
@Override
public Relation compare(byte[] minPacked, byte[] maxPacked) {
boolean crosses = false;
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
if (FutureArrays.compareUnsigned(maxPacked, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMin[dim], 0, numBytesPerDim) < 0 ||
FutureArrays.compareUnsigned(minPacked, dim * numBytesPerDim, dim * numBytesPerDim + numBytesPerDim, queryMax[dim], 0, numBytesPerDim) > 0) {
return Relation.CELL_OUTSIDE_QUERY;
@@ -800,7 +808,7 @@ public class TestBKD extends LuceneTestCase {
BitSet expected = new BitSet();
for(int ord=0;ord<numValues;ord++) {
boolean matches = true;
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
byte[] x = docValues[ord][dim];
if (FutureArrays.compareUnsigned(x, 0, numBytesPerDim, queryMin[dim], 0, numBytesPerDim) < 0 ||
FutureArrays.compareUnsigned(x, 0, numBytesPerDim, queryMax[dim], 0, numBytesPerDim) > 0) {
@@ -897,7 +905,7 @@ public class TestBKD extends LuceneTestCase {
};
CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
- verify(dir, docValues, null, numDims, numBytesPerDim, 50, 0.1);
+ verify(dir, docValues, null, numDims, numDims, numBytesPerDim, 50, 0.1);
});
assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
}
@@ -945,7 +953,7 @@ public class TestBKD extends LuceneTestCase {
};
Throwable t = expectThrows(CorruptIndexException.class, () -> {
- verify(dir, docValues, null, numDims, numBytesPerDim, 50, 0.1);
+ verify(dir, docValues, null, numDims, numDims, numBytesPerDim, 50, 0.1);
});
assertCorruptionDetected(t);
}
@@ -971,7 +979,7 @@ public class TestBKD extends LuceneTestCase {
public void testTieBreakOrder() throws Exception {
try (Directory dir = newDirectory()) {
int numDocs = 10000;
- BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", 1, Integer.BYTES, 2, 0.01f, numDocs, true);
+ BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", 1, 1, Integer.BYTES, 2, 0.01f, numDocs, true);
for(int i=0;i<numDocs;i++) {
w.add(new byte[Integer.BYTES], i);
}
@@ -1012,7 +1020,7 @@ public class TestBKD extends LuceneTestCase {
boolean singleValuePerDoc = false;
boolean longOrds = true;
int offlineSorterMaxTempFiles = TestUtil.nextInt(random(), 2, 20);
- BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", 2, Integer.BYTES, 2, 0.01f, numDocs,
+ BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", 2, 2, Integer.BYTES, 2, 0.01f, numDocs,
singleValuePerDoc, longOrds, 1, offlineSorterMaxTempFiles);
byte[] buffer = new byte[2*Integer.BYTES];
for(int i=0;i<numDocs;i++) {
@@ -1058,12 +1066,13 @@ public class TestBKD extends LuceneTestCase {
// values as a LongPoint:
public void testWastedLeadingBytes() throws Exception {
int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS);
+ int numIndexDims = TestUtil.nextInt(random(), 1, numDims);
int bytesPerDim = PointValues.MAX_NUM_BYTES;
int bytesUsed = TestUtil.nextInt(random(), 1, 3);
Directory dir = newFSDirectory(createTempDir());
int numDocs = 100000;
- BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", numDims, bytesPerDim, 32, 1f, numDocs, true);
+ BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", numDims, numIndexDims, bytesPerDim, 32, 1f, numDocs, true);
byte[] tmp = new byte[bytesUsed];
byte[] buffer = new byte[numDims * bytesPerDim];
for(int i=0;i<numDocs;i++) {
@@ -1091,11 +1100,14 @@ public class TestBKD extends LuceneTestCase {
@Override
public void visit(int docID, byte[] packedValue) {
+ assert packedValue.length == numDims * bytesPerDim;
visit(docID);
}
@Override
public Relation compare(byte[] minPacked, byte[] maxPacked) {
+ assert minPacked.length == numIndexDims * bytesPerDim;
+ assert maxPacked.length == numIndexDims * bytesPerDim;
if (random().nextInt(7) == 1) {
return Relation.CELL_CROSSES_QUERY;
} else {
@@ -1117,7 +1129,7 @@ public class TestBKD extends LuceneTestCase {
final byte[] uniquePointValue = new byte[numBytesPerDim];
random().nextBytes(uniquePointValue);
- BKDWriter w = new BKDWriter(numValues, dir, "_temp", 1, numBytesPerDim, maxPointsInLeafNode,
+ BKDWriter w = new BKDWriter(numValues, dir, "_temp", 1, 1, numBytesPerDim, maxPointsInLeafNode,
BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, numValues, true);
for (int i = 0; i < numValues; ++i) {
if (i == numValues / 2) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/test/org/apache/lucene/util/bkd/TestMutablePointsReaderUtils.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestMutablePointsReaderUtils.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestMutablePointsReaderUtils.java
index f38ab5a..da75b12 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestMutablePointsReaderUtils.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestMutablePointsReaderUtils.java
@@ -232,7 +232,12 @@ public class TestMutablePointsReaderUtils extends LuceneTestCase {
}
@Override
- public int getNumDimensions() throws IOException {
+ public int getNumDataDimensions() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getNumIndexDimensions() throws IOException {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
index 1eef95f..42f7aec 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
@@ -81,7 +81,7 @@ public class TermVectorLeafReader extends LeafReader {
}
FieldInfo fieldInfo = new FieldInfo(field, 0,
true, true, terms.hasPayloads(),
- indexOptions, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, false);
+ indexOptions, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false);
fieldInfos = new FieldInfos(new FieldInfo[]{fieldInfo});
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java
index 02b7e86..80810e2 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java
@@ -146,8 +146,8 @@ abstract class PointInSetIncludingScoreQuery extends Query {
if (fieldInfo == null) {
return null;
}
- if (fieldInfo.getPointDimensionCount() != 1) {
- throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numDims=" + fieldInfo.getPointDimensionCount() + " but this query has numDims=1");
+ if (fieldInfo.getPointDataDimensionCount() != 1) {
+ throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numDims=" + fieldInfo.getPointDataDimensionCount() + " but this query has numDims=1");
}
if (fieldInfo.getPointNumBytes() != bytesPerDim) {
throw new IllegalArgumentException("field=\"" + field + "\" was indexed with bytesPerDim=" + fieldInfo.getPointNumBytes() + " but this query has bytesPerDim=" + bytesPerDim);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index 42af398..0078c3f 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -410,7 +410,7 @@ public class MemoryIndex {
storeDocValues(info, docValuesType, docValuesValue);
}
- if (field.fieldType().pointDimensionCount() > 0) {
+ if (field.fieldType().pointDataDimensionCount() > 0) {
storePointValues(info, field.binaryValue());
}
@@ -486,9 +486,9 @@ public class MemoryIndex {
if (info == null) {
fields.put(fieldName, info = new Info(createFieldInfo(fieldName, fields.size(), fieldType), byteBlockPool));
}
- if (fieldType.pointDimensionCount() != info.fieldInfo.getPointDimensionCount()) {
- if (fieldType.pointDimensionCount() > 0)
- info.fieldInfo.setPointDimensions(fieldType.pointDimensionCount(), fieldType.pointNumBytes());
+ if (fieldType.pointDataDimensionCount() != info.fieldInfo.getPointDataDimensionCount()) {
+ if (fieldType.pointDataDimensionCount() > 0)
+ info.fieldInfo.setPointDimensions(fieldType.pointDataDimensionCount(), fieldType.pointIndexDimensionCount(), fieldType.pointNumBytes());
}
if (fieldType.docValuesType() != info.fieldInfo.getDocValuesType()) {
if (fieldType.docValuesType() != DocValuesType.NONE)
@@ -501,7 +501,7 @@ public class MemoryIndex {
IndexOptions indexOptions = storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
return new FieldInfo(fieldName, ord, fieldType.storeTermVectors(), fieldType.omitNorms(), storePayloads,
indexOptions, fieldType.docValuesType(), -1, Collections.emptyMap(),
- fieldType.pointDimensionCount(), fieldType.pointNumBytes(), false);
+ fieldType.pointDataDimensionCount(), fieldType.pointIndexDimensionCount(), fieldType.pointNumBytes(), false);
}
private void storePointValues(Info info, BytesRef pointValue) {
@@ -520,7 +520,8 @@ public class MemoryIndex {
info.fieldInfo = new FieldInfo(
info.fieldInfo.name, info.fieldInfo.number, info.fieldInfo.hasVectors(), info.fieldInfo.hasPayloads(),
info.fieldInfo.hasPayloads(), info.fieldInfo.getIndexOptions(), docValuesType, -1, info.fieldInfo.attributes(),
- info.fieldInfo.getPointDimensionCount(), info.fieldInfo.getPointNumBytes(), info.fieldInfo.isSoftDeletesField()
+ info.fieldInfo.getPointDataDimensionCount(), info.fieldInfo.getPointIndexDimensionCount(), info.fieldInfo.getPointNumBytes(),
+ info.fieldInfo.isSoftDeletesField()
);
} else if (existingDocValuesType != docValuesType) {
throw new IllegalArgumentException("Can't add [" + docValuesType + "] doc values field [" + fieldName + "], because [" + existingDocValuesType + "] doc values field already exists");
@@ -870,7 +871,7 @@ public class MemoryIndex {
if (pointValues != null) {
assert pointValues[0].bytes.length == pointValues[0].length : "BytesRef should wrap a precise byte[], BytesRef.deepCopyOf() should take care of this";
- final int numDimensions = fieldInfo.getPointDimensionCount();
+ final int numDimensions = fieldInfo.getPointDataDimensionCount();
final int numBytesPerDimension = fieldInfo.getPointNumBytes();
if (numDimensions == 1) {
// PointInSetQuery.MergePointVisitor expects values to be visited in increasing order,
@@ -1576,8 +1577,13 @@ public class MemoryIndex {
}
@Override
- public int getNumDimensions() throws IOException {
- return info.fieldInfo.getPointDimensionCount();
+ public int getNumDataDimensions() throws IOException {
+ return info.fieldInfo.getPointDataDimensionCount();
+ }
+
+ @Override
+ public int getNumIndexDimensions() throws IOException {
+ return info.fieldInfo.getPointDataDimensionCount();
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java
index 4dc50d1..82eeb3f 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java
@@ -62,8 +62,8 @@ public class BigIntegerPoint extends Field {
/** Change the values of this field */
public void setBigIntegerValues(BigInteger... point) {
- if (type.pointDimensionCount() != point.length) {
- throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
+ if (type.pointDataDimensionCount() != point.length) {
+ throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
}
fieldsData = pack(point);
}
@@ -75,8 +75,8 @@ public class BigIntegerPoint extends Field {
@Override
public Number numericValue() {
- if (type.pointDimensionCount() != 1) {
- throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value");
+ if (type.pointDataDimensionCount() != 1) {
+ throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot convert to a single numeric value");
}
BytesRef bytes = (BytesRef) fieldsData;
assert bytes.length == BYTES;
@@ -119,7 +119,7 @@ public class BigIntegerPoint extends Field {
result.append(':');
BytesRef bytes = (BytesRef) fieldsData;
- for (int dim = 0; dim < type.pointDimensionCount(); dim++) {
+ for (int dim = 0; dim < type.pointDataDimensionCount(); dim++) {
if (dim > 0) {
result.append(',');
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/sandbox/src/java/org/apache/lucene/document/HalfFloatPoint.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/HalfFloatPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/HalfFloatPoint.java
index 1f599a6..15d8240 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/HalfFloatPoint.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/HalfFloatPoint.java
@@ -213,8 +213,8 @@ public final class HalfFloatPoint extends Field {
/** Change the values of this field */
public void setFloatValues(float... point) {
- if (type.pointDimensionCount() != point.length) {
- throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
+ if (type.pointDataDimensionCount() != point.length) {
+ throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
}
fieldsData = pack(point);
}
@@ -226,8 +226,8 @@ public final class HalfFloatPoint extends Field {
@Override
public Number numericValue() {
- if (type.pointDimensionCount() != 1) {
- throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value");
+ if (type.pointDataDimensionCount() != 1) {
+ throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot convert to a single numeric value");
}
BytesRef bytes = (BytesRef) fieldsData;
assert bytes.length == BYTES;
@@ -270,7 +270,7 @@ public final class HalfFloatPoint extends Field {
result.append(':');
BytesRef bytes = (BytesRef) fieldsData;
- for (int dim = 0; dim < type.pointDimensionCount(); dim++) {
+ for (int dim = 0; dim < type.pointDataDimensionCount(); dim++) {
if (dim > 0) {
result.append(',');
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
index 5029d0e..2cdef86 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
@@ -148,7 +148,7 @@ public class BBoxStrategy extends SpatialStrategy {
if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
numQuads++;
}
- if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
+ if ((this.hasPointVals = fieldType.pointDataDimensionCount() > 0)) {
numQuads++;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java
index c7904df..f06770e 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java
@@ -139,7 +139,7 @@ public class PointVectorStrategy extends SpatialStrategy {
if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
numPairs++;
}
- if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
+ if ((this.hasPointVals = fieldType.pointDataDimensionCount() > 0)) {
numPairs++;
}
this.fieldsLen = numPairs * 2;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
index ff2e1b6..4943b99 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
@@ -131,8 +131,8 @@ public final class AssertingPointsFormat extends PointsFormat {
@Override
public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException {
- if (fieldInfo.getPointDimensionCount() == 0) {
- throw new IllegalArgumentException("writing field=\"" + fieldInfo.name + "\" but pointDimensionalCount is 0");
+ if (fieldInfo.getPointDataDimensionCount() == 0) {
+ throw new IllegalArgumentException("writing field=\"" + fieldInfo.name + "\" but pointDataDimensionalCount is 0");
}
in.writeField(fieldInfo, values);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java
index 486d81c..681edbd 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java
@@ -155,11 +155,19 @@ class CrankyPointsFormat extends PointsFormat {
}
@Override
- public int getNumDimensions() throws IOException {
+ public int getNumDataDimensions() throws IOException {
if (random.nextInt(100) == 0) {
throw new IOException("Fake IOException");
}
- return delegate.getNumDimensions();
+ return delegate.getNumDataDimensions();
+ }
+
+ @Override
+ public int getNumIndexDimensions() throws IOException {
+ if (random.nextInt(100) == 0) {
+ throw new IOException("Fake IOException");
+ }
+ return delegate.getNumIndexDimensions();
}
@Override
[4/4] lucene-solr:master: LUCENE-8496: Selective indexing - modify
BKDReader/BKDWriter to allow users to select a fewer number of dimensions to
be used for creating the index than the total number of dimensions used for
field encoding. i.e., dimensions 0
Posted by nk...@apache.org.
LUCENE-8496: Selective indexing - modify BKDReader/BKDWriter to allow users to select a fewer number of dimensions to be used for creating the index than the total number of dimensions used for field encoding. i.e., dimensions 0 to N may be used to determine how to split the inner nodes, and dimensions N+1 to D are ignored and stored as data dimensions at the leaves.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1118299c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1118299c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1118299c
Branch: refs/heads/master
Commit: 1118299c338253cea09640acdc48dc930dc27fda
Parents: df07a43
Author: Nicholas Knize <nk...@gmail.com>
Authored: Mon Oct 8 18:51:03 2018 -0500
Committer: Nicholas Knize <nk...@gmail.com>
Committed: Mon Oct 8 18:51:03 2018 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 8 +
.../codecs/simpletext/SimpleTextBKDReader.java | 59 ++++---
.../codecs/simpletext/SimpleTextBKDWriter.java | 144 ++++++++-------
.../simpletext/SimpleTextFieldInfosFormat.java | 21 ++-
.../simpletext/SimpleTextPointsReader.java | 20 ++-
.../simpletext/SimpleTextPointsWriter.java | 6 +-
.../org/apache/lucene/codecs/PointsWriter.java | 13 +-
.../lucene50/Lucene50FieldInfosFormat.java | 2 +-
.../lucene60/Lucene60FieldInfosFormat.java | 21 ++-
.../codecs/lucene60/Lucene60PointsReader.java | 2 +-
.../codecs/lucene60/Lucene60PointsWriter.java | 14 +-
.../codecs/perfield/PerFieldMergeState.java | 2 +-
.../org/apache/lucene/document/BinaryPoint.java | 4 +-
.../org/apache/lucene/document/DoublePoint.java | 10 +-
.../org/apache/lucene/document/DoubleRange.java | 10 +-
.../org/apache/lucene/document/FieldType.java | 66 +++++--
.../org/apache/lucene/document/FloatPoint.java | 10 +-
.../org/apache/lucene/document/FloatRange.java | 10 +-
.../org/apache/lucene/document/IntPoint.java | 10 +-
.../org/apache/lucene/document/IntRange.java | 10 +-
.../org/apache/lucene/document/LatLonPoint.java | 6 +-
.../org/apache/lucene/document/LongPoint.java | 10 +-
.../org/apache/lucene/document/LongRange.java | 10 +-
.../apache/lucene/document/RangeFieldQuery.java | 4 +-
.../org/apache/lucene/index/CheckIndex.java | 37 ++--
.../org/apache/lucene/index/CodecReader.java | 2 +-
.../lucene/index/DefaultIndexingChain.java | 15 +-
.../java/org/apache/lucene/index/FieldInfo.java | 73 +++++---
.../org/apache/lucene/index/FieldInfos.java | 61 ++++---
.../org/apache/lucene/index/IndexWriter.java | 6 +-
.../apache/lucene/index/IndexableFieldType.java | 9 +-
.../org/apache/lucene/index/PointValues.java | 11 +-
.../apache/lucene/index/PointValuesWriter.java | 20 ++-
.../apache/lucene/index/SortingLeafReader.java | 9 +-
.../apache/lucene/search/PointInSetQuery.java | 4 +-
.../apache/lucene/search/PointRangeQuery.java | 4 +-
.../org/apache/lucene/util/bkd/BKDReader.java | 140 ++++++++-------
.../org/apache/lucene/util/bkd/BKDWriter.java | 174 ++++++++++---------
.../apache/lucene/document/TestFieldType.java | 11 +-
.../apache/lucene/index/TestIndexableField.java | 7 +-
.../lucene/index/TestPendingSoftDeletes.java | 10 +-
.../apache/lucene/index/TestPointValues.java | 14 +-
.../apache/lucene/search/TestPointQueries.java | 2 +-
.../apache/lucene/util/TestDocIdSetBuilder.java | 7 +-
.../apache/lucene/util/bkd/Test2BBKDPoints.java | 4 +-
.../org/apache/lucene/util/bkd/TestBKD.java | 142 ++++++++-------
.../util/bkd/TestMutablePointsReaderUtils.java | 7 +-
.../search/highlight/TermVectorLeafReader.java | 2 +-
.../join/PointInSetIncludingScoreQuery.java | 4 +-
.../apache/lucene/index/memory/MemoryIndex.java | 24 ++-
.../apache/lucene/document/BigIntegerPoint.java | 10 +-
.../apache/lucene/document/HalfFloatPoint.java | 10 +-
.../lucene/spatial/bbox/BBoxStrategy.java | 2 +-
.../spatial/vector/PointVectorStrategy.java | 2 +-
.../codecs/asserting/AssertingPointsFormat.java | 4 +-
.../codecs/cranky/CrankyPointsFormat.java | 12 +-
.../lucene/index/AssertingLeafReader.java | 41 +++--
.../index/BaseIndexFileFormatTestCase.java | 2 +-
.../lucene/index/BasePointsFormatTestCase.java | 76 +++++---
.../lucene/index/MismatchedLeafReader.java | 3 +-
.../org/apache/lucene/index/RandomCodec.java | 9 +-
.../lucene/index/RandomPostingsTester.java | 4 +-
.../org/apache/lucene/util/LuceneTestCase.java | 16 +-
.../java/org/apache/lucene/util/TestUtil.java | 2 +-
.../solr/handler/component/ExpandComponent.java | 3 +-
.../org/apache/solr/legacy/BBoxStrategy.java | 2 +-
.../apache/solr/legacy/PointVectorStrategy.java | 2 +-
.../org/apache/solr/schema/SchemaField.java | 7 +-
.../solr/search/CollapsingQParserPlugin.java | 2 +-
.../java/org/apache/solr/search/Insanity.java | 2 +-
.../apache/solr/uninverting/FieldCacheImpl.java | 6 +-
.../solr/uninverting/UninvertingReader.java | 6 +-
.../solr/uninverting/TestUninvertingReader.java | 6 +-
73 files changed, 892 insertions(+), 608 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index ad83ff9..80d341e 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -192,6 +192,14 @@ Bug fixes:
* LUCENE-8479: QueryBuilder#analyzeGraphPhrase now throws TooManyClause exception
if the number of expanded path reaches the BooleanQuery#maxClause limit. (Jim Ferenczi)
+New Features
+
+* LUCENE-8496: Selective indexing - modify BKDReader/BKDWriter to allow users
+ to select a fewer number of dimensions to be used for creating the index than
+ the total number of dimensions used for field encoding. i.e., dimensions 0 to N
+ may be used to determine how to split the inner nodes, and dimensions N+1 to D
+ are ignored and stored as data dimensions at the leaves. (Nick Knize)
+
======================= Lucene 7.5.1 =======================
Bug Fixes:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDReader.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDReader.java
index b7af45a..5227d6d 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDReader.java
@@ -41,7 +41,8 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
final private byte[] splitPackedValues;
final long[] leafBlockFPs;
final private int leafNodeOffset;
- final int numDims;
+ final int numDataDims;
+ final int numIndexDims;
final int bytesPerDim;
final int bytesPerIndexEntry;
final IndexInput in;
@@ -52,16 +53,19 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
final int docCount;
final int version;
protected final int packedBytesLength;
+ protected final int packedIndexBytesLength;
- public SimpleTextBKDReader(IndexInput in, int numDims, int maxPointsInLeafNode, int bytesPerDim, long[] leafBlockFPs, byte[] splitPackedValues,
+ public SimpleTextBKDReader(IndexInput in, int numDataDims, int numIndexDims, int maxPointsInLeafNode, int bytesPerDim, long[] leafBlockFPs, byte[] splitPackedValues,
byte[] minPackedValue, byte[] maxPackedValue, long pointCount, int docCount) throws IOException {
this.in = in;
- this.numDims = numDims;
+ this.numDataDims = numDataDims;
+ this.numIndexDims = numIndexDims;
this.maxPointsInLeafNode = maxPointsInLeafNode;
this.bytesPerDim = bytesPerDim;
// no version check here because callers of this API (SimpleText) have no back compat:
- bytesPerIndexEntry = numDims == 1 ? bytesPerDim : bytesPerDim + 1;
- packedBytesLength = numDims * bytesPerDim;
+ bytesPerIndexEntry = numIndexDims == 1 ? bytesPerDim : bytesPerDim + 1;
+ packedBytesLength = numDataDims * bytesPerDim;
+ packedIndexBytesLength = numIndexDims * bytesPerDim;
this.leafNodeOffset = leafBlockFPs.length;
this.leafBlockFPs = leafBlockFPs;
this.splitPackedValues = splitPackedValues;
@@ -70,8 +74,8 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
this.pointCount = pointCount;
this.docCount = docCount;
this.version = SimpleTextBKDWriter.VERSION_CURRENT;
- assert minPackedValue.length == packedBytesLength;
- assert maxPackedValue.length == packedBytesLength;
+ assert minPackedValue.length == packedIndexBytesLength;
+ assert maxPackedValue.length == packedIndexBytesLength;
}
/** Used to track all state for a single call to {@link #intersect}. */
@@ -115,7 +119,7 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
/** Create a new {@link IntersectState} */
public IntersectState getIntersectState(IntersectVisitor visitor) {
- return new IntersectState(in.clone(), numDims,
+ return new IntersectState(in.clone(), numDataDims,
packedBytesLength,
maxPointsInLeafNode,
visitor);
@@ -181,7 +185,7 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
scratchPackedValue[compressedByteOffset] = in.readByte();
final int runLen = Byte.toUnsignedInt(in.readByte());
for (int j = 0; j < runLen; ++j) {
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
int prefix = commonPrefixLengths[dim];
in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix);
}
@@ -196,14 +200,14 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
private int readCompressedDim(IndexInput in) throws IOException {
int compressedDim = in.readByte();
- if (compressedDim < -1 || compressedDim >= numDims) {
+ if (compressedDim < -1 || compressedDim >= numIndexDims) {
throw new CorruptIndexException("Got compressedDim="+compressedDim, in);
}
return compressedDim;
}
private void readCommonPrefixes(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in) throws IOException {
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
int prefix = in.readVInt();
commonPrefixLengths[dim] = prefix;
if (prefix > 0) {
@@ -258,27 +262,27 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
int address = nodeID * bytesPerIndexEntry;
int splitDim;
- if (numDims == 1) {
+ if (numIndexDims == 1) {
splitDim = 0;
} else {
splitDim = splitPackedValues[address++] & 0xff;
}
- assert splitDim < numDims;
+ assert splitDim < numIndexDims;
// TODO: can we alloc & reuse this up front?
- byte[] splitPackedValue = new byte[packedBytesLength];
+ byte[] splitPackedValue = new byte[packedIndexBytesLength];
// Recurse on left sub-tree:
- System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedBytesLength);
+ System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitPackedValues, address, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
intersect(state,
2*nodeID,
cellMinPacked, splitPackedValue);
// Recurse on right sub-tree:
- System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedBytesLength);
+ System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitPackedValues, address, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
intersect(state,
2*nodeID+1,
@@ -307,27 +311,27 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
int address = nodeID * bytesPerIndexEntry;
int splitDim;
- if (numDims == 1) {
+ if (numIndexDims == 1) {
splitDim = 0;
} else {
splitDim = splitPackedValues[address++] & 0xff;
}
- assert splitDim < numDims;
+ assert splitDim < numIndexDims;
// TODO: can we alloc & reuse this up front?
- byte[] splitPackedValue = new byte[packedBytesLength];
+ byte[] splitPackedValue = new byte[packedIndexBytesLength];
// Recurse on left sub-tree:
- System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedBytesLength);
+ System.arraycopy(cellMaxPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitPackedValues, address, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
final long leftCost = estimatePointCount(state,
2*nodeID,
cellMinPacked, splitPackedValue);
// Recurse on right sub-tree:
- System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedBytesLength);
+ System.arraycopy(cellMinPacked, 0, splitPackedValue, 0, packedIndexBytesLength);
System.arraycopy(splitPackedValues, address, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
final long rightCost = estimatePointCount(state,
2*nodeID+1,
@@ -340,13 +344,13 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
public void copySplitValue(int nodeID, byte[] splitPackedValue) {
int address = nodeID * bytesPerIndexEntry;
int splitDim;
- if (numDims == 1) {
+ if (numIndexDims == 1) {
splitDim = 0;
} else {
splitDim = splitPackedValues[address++] & 0xff;
}
- assert splitDim < numDims;
+ assert splitDim < numIndexDims;
System.arraycopy(splitPackedValues, address, splitPackedValue, splitDim*bytesPerDim, bytesPerDim);
}
@@ -367,8 +371,13 @@ final class SimpleTextBKDReader extends PointValues implements Accountable {
}
@Override
- public int getNumDimensions() {
- return numDims;
+ public int getNumDataDimensions() {
+ return numDataDims;
+ }
+
+ @Override
+ public int getNumIndexDimensions() {
+ return numIndexDims;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDWriter.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDWriter.java
index 1ddc111..f45209c 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDWriter.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextBKDWriter.java
@@ -62,7 +62,8 @@ import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.INDEX_C
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.MAX_LEAF_POINTS;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.MAX_VALUE;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.MIN_VALUE;
-import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.NUM_DIMS;
+import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.NUM_DATA_DIMS;
+import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.NUM_INDEX_DIMS;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.POINT_COUNT;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.SPLIT_COUNT;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.SPLIT_DIM;
@@ -102,15 +103,21 @@ final class SimpleTextBKDWriter implements Closeable {
/** Maximum number of dimensions */
public static final int MAX_DIMS = 8;
- /** How many dimensions we are indexing */
- protected final int numDims;
+ /** How many dimensions we are storing at the leaf (data) nodes */
+ protected final int numDataDims;
+
+ /** How many dimensions we are indexing in the internal nodes */
+ protected final int numIndexDims;
/** How many bytes each value in each dimension takes. */
protected final int bytesPerDim;
- /** numDims * bytesPerDim */
+ /** numDataDims * bytesPerDim */
protected final int packedBytesLength;
+ /** numIndexDims * bytesPerDim */
+ protected final int packedIndexBytesLength;
+
final BytesRefBuilder scratch = new BytesRefBuilder();
final TrackingDirectoryWrapper tempDir;
@@ -158,37 +165,39 @@ final class SimpleTextBKDWriter implements Closeable {
private final int maxDoc;
- public SimpleTextBKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim,
+ public SimpleTextBKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDataDims, int numIndexDims, int bytesPerDim,
int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount, boolean singleValuePerDoc) throws IOException {
- this(maxDoc, tempDir, tempFileNamePrefix, numDims, bytesPerDim, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount, singleValuePerDoc,
+ this(maxDoc, tempDir, tempFileNamePrefix, numDataDims, numIndexDims, bytesPerDim, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount, singleValuePerDoc,
totalPointCount > Integer.MAX_VALUE, Math.max(1, (long) maxMBSortInHeap), OfflineSorter.MAX_TEMPFILES);
}
- private SimpleTextBKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim,
+ private SimpleTextBKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDataDims, int numIndexDims, int bytesPerDim,
int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount,
boolean singleValuePerDoc, boolean longOrds, long offlineSorterBufferMB, int offlineSorterMaxTempFiles) throws IOException {
- verifyParams(numDims, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount);
+ verifyParams(numDataDims, numIndexDims, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount);
// We use tracking dir to deal with removing files on exception, so each place that
// creates temp files doesn't need crazy try/finally/sucess logic:
this.tempDir = new TrackingDirectoryWrapper(tempDir);
this.tempFileNamePrefix = tempFileNamePrefix;
this.maxPointsInLeafNode = maxPointsInLeafNode;
- this.numDims = numDims;
+ this.numDataDims = numDataDims;
+ this.numIndexDims = numIndexDims;
this.bytesPerDim = bytesPerDim;
this.totalPointCount = totalPointCount;
this.maxDoc = maxDoc;
this.offlineSorterBufferMB = OfflineSorter.BufferSize.megabytes(offlineSorterBufferMB);
this.offlineSorterMaxTempFiles = offlineSorterMaxTempFiles;
docsSeen = new FixedBitSet(maxDoc);
- packedBytesLength = numDims * bytesPerDim;
+ packedBytesLength = numDataDims * bytesPerDim;
+ packedIndexBytesLength = numIndexDims * bytesPerDim;
scratchDiff = new byte[bytesPerDim];
scratch1 = new byte[packedBytesLength];
scratch2 = new byte[packedBytesLength];
- commonPrefixLengths = new int[numDims];
+ commonPrefixLengths = new int[numDataDims];
- minPackedValue = new byte[packedBytesLength];
- maxPackedValue = new byte[packedBytesLength];
+ minPackedValue = new byte[packedIndexBytesLength];
+ maxPackedValue = new byte[packedIndexBytesLength];
// If we may have more than 1+Integer.MAX_VALUE values, then we must encode ords with long (8 bytes), else we can use int (4 bytes).
this.longOrds = longOrds;
@@ -215,7 +224,7 @@ final class SimpleTextBKDWriter implements Closeable {
// bytes to points here. Each dimension has its own sorted partition, so
// we must divide by numDims as wel.
- maxPointsSortInHeap = (int) (0.5 * (maxMBSortInHeap * 1024 * 1024) / (bytesPerDoc * numDims));
+ maxPointsSortInHeap = (int) (0.5 * (maxMBSortInHeap * 1024 * 1024) / (bytesPerDoc * numDataDims));
// Finally, we must be able to hold at least the leaf node in heap during build:
if (maxPointsSortInHeap < maxPointsInLeafNode) {
@@ -228,11 +237,14 @@ final class SimpleTextBKDWriter implements Closeable {
this.maxMBSortInHeap = maxMBSortInHeap;
}
- public static void verifyParams(int numDims, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount) {
+ public static void verifyParams(int numDataDims, int numIndexDims, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount) {
// We encode dim in a single byte in the splitPackedValues, but we only expose 4 bits for it now, in case we want to use
// remaining 4 bits for another purpose later
- if (numDims < 1 || numDims > MAX_DIMS) {
- throw new IllegalArgumentException("numDims must be 1 .. " + MAX_DIMS + " (got: " + numDims + ")");
+ if (numDataDims < 1 || numDataDims > MAX_DIMS) {
+ throw new IllegalArgumentException("numDataDims must be 1 .. " + MAX_DIMS + " (got: " + numDataDims + ")");
+ }
+ if (numIndexDims < 1 || numIndexDims > numDataDims) {
+ throw new IllegalArgumentException("numIndexDims must be 1 .. " + numDataDims + " (got: " + numIndexDims + ")");
}
if (maxPointsInLeafNode <= 0) {
throw new IllegalArgumentException("maxPointsInLeafNode must be > 0; got " + maxPointsInLeafNode);
@@ -281,10 +293,10 @@ final class SimpleTextBKDWriter implements Closeable {
// TODO: we could specialize for the 1D case:
if (pointCount == 0) {
- System.arraycopy(packedValue, 0, minPackedValue, 0, packedBytesLength);
- System.arraycopy(packedValue, 0, maxPackedValue, 0, packedBytesLength);
+ System.arraycopy(packedValue, 0, minPackedValue, 0, packedIndexBytesLength);
+ System.arraycopy(packedValue, 0, maxPackedValue, 0, packedIndexBytesLength);
} else {
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
int offset = dim*bytesPerDim;
if (FutureArrays.compareUnsigned(packedValue, offset, offset + bytesPerDim, minPackedValue, offset, offset + bytesPerDim) < 0) {
System.arraycopy(packedValue, offset, minPackedValue, offset, bytesPerDim);
@@ -313,7 +325,7 @@ final class SimpleTextBKDWriter implements Closeable {
* disk. This method does not use transient disk in order to reorder points.
*/
public long writeField(IndexOutput out, String fieldName, MutablePointValues reader) throws IOException {
- if (numDims == 1) {
+ if (numIndexDims == 1) {
return writeField1Dim(out, fieldName, reader);
} else {
return writeFieldNDims(out, fieldName, reader);
@@ -356,7 +368,7 @@ final class SimpleTextBKDWriter implements Closeable {
Arrays.fill(maxPackedValue, (byte) 0);
for (int i = 0; i < Math.toIntExact(pointCount); ++i) {
values.getValue(i, scratchBytesRef1);
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
int offset = dim*bytesPerDim;
if (FutureArrays.compareUnsigned(scratchBytesRef1.bytes, scratchBytesRef1.offset + offset, scratchBytesRef1.offset + offset + bytesPerDim, minPackedValue, offset, offset + bytesPerDim) < 0) {
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + offset, minPackedValue, offset, bytesPerDim);
@@ -382,7 +394,7 @@ final class SimpleTextBKDWriter implements Closeable {
/* In the 1D case, we can simply sort points in ascending order and use the
* same writing logic as we use at merge time. */
private long writeField1Dim(IndexOutput out, String fieldName, MutablePointValues reader) throws IOException {
- MutablePointsReaderUtils.sort(maxDoc, packedBytesLength, reader, 0, Math.toIntExact(reader.size()));
+ MutablePointsReaderUtils.sort(maxDoc, packedIndexBytesLength, reader, 0, Math.toIntExact(reader.size()));
final OneDimensionBKDWriter oneDimWriter = new OneDimensionBKDWriter(out);
@@ -418,8 +430,8 @@ final class SimpleTextBKDWriter implements Closeable {
int leafCount;
OneDimensionBKDWriter(IndexOutput out) {
- if (numDims != 1) {
- throw new UnsupportedOperationException("numDims must be 1 but got " + numDims);
+ if (numIndexDims != 1) {
+ throw new UnsupportedOperationException("numIndexDims must be 1 but got " + numIndexDims);
}
if (pointCount != 0) {
throw new IllegalStateException("cannot mix add and merge");
@@ -497,9 +509,9 @@ final class SimpleTextBKDWriter implements Closeable {
private void writeLeafBlock() throws IOException {
assert leafCount != 0;
if (valueCount == 0) {
- System.arraycopy(leafValues, 0, minPackedValue, 0, packedBytesLength);
+ System.arraycopy(leafValues, 0, minPackedValue, 0, packedIndexBytesLength);
}
- System.arraycopy(leafValues, (leafCount - 1) * packedBytesLength, maxPackedValue, 0, packedBytesLength);
+ System.arraycopy(leafValues, (leafCount - 1) * packedBytesLength, maxPackedValue, 0, packedIndexBytesLength);
valueCount += leafCount;
@@ -512,7 +524,7 @@ final class SimpleTextBKDWriter implements Closeable {
Arrays.fill(commonPrefixLengths, bytesPerDim);
// Find per-dim common prefix:
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
int offset1 = dim * bytesPerDim;
int offset2 = (leafCount - 1) * packedBytesLength + offset1;
for(int j=0;j<commonPrefixLengths[dim];j++) {
@@ -658,7 +670,7 @@ final class SimpleTextBKDWriter implements Closeable {
}
private PointWriter sort(int dim) throws IOException {
- assert dim >= 0 && dim < numDims;
+ assert dim >= 0 && dim < numDataDims;
if (heapPointWriter != null) {
@@ -691,7 +703,7 @@ final class SimpleTextBKDWriter implements Closeable {
final int offset = bytesPerDim * dim;
Comparator<BytesRef> cmp;
- if (dim == numDims - 1) {
+ if (dim == numDataDims - 1) {
// in that case the bytes for the dimension and for the doc id are contiguous,
// so we don't need a branch
cmp = new BytesRefComparator(bytesPerDim + Integer.BYTES) {
@@ -776,7 +788,7 @@ final class SimpleTextBKDWriter implements Closeable {
}
LongBitSet ordBitSet;
- if (numDims > 1) {
+ if (numDataDims > 1) {
if (singleValuePerDoc) {
ordBitSet = new LongBitSet(maxDoc);
} else {
@@ -811,7 +823,7 @@ final class SimpleTextBKDWriter implements Closeable {
assert pointCount / numLeaves <= maxPointsInLeafNode: "pointCount=" + pointCount + " numLeaves=" + numLeaves + " maxPointsInLeafNode=" + maxPointsInLeafNode;
// Sort all docs once by each dimension:
- PathSlice[] sortedPointWriters = new PathSlice[numDims];
+ PathSlice[] sortedPointWriters = new PathSlice[numDataDims];
// This is only used on exception; on normal code paths we close all files we opened:
List<Closeable> toCloseHeroically = new ArrayList<>();
@@ -819,7 +831,7 @@ final class SimpleTextBKDWriter implements Closeable {
boolean success = false;
try {
//long t0 = System.nanoTime();
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
sortedPointWriters[dim] = new PathSlice(sort(dim), 0, pointCount);
}
//long t1 = System.nanoTime();
@@ -867,8 +879,12 @@ final class SimpleTextBKDWriter implements Closeable {
/** Subclass can change how it writes the index. */
private void writeIndex(IndexOutput out, long[] leafBlockFPs, byte[] splitPackedValues) throws IOException {
- write(out, NUM_DIMS);
- writeInt(out, numDims);
+ write(out, NUM_DATA_DIMS);
+ writeInt(out, numDataDims);
+ newline(out);
+
+ write(out, NUM_INDEX_DIMS);
+ writeInt(out, numIndexDims);
newline(out);
write(out, BYTES_PER_DIM);
@@ -952,7 +968,7 @@ final class SimpleTextBKDWriter implements Closeable {
BytesRef ref = packedValues.apply(i);
assert ref.length == packedBytesLength;
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
int prefix = commonPrefixLengths[dim];
out.writeBytes(ref.bytes, ref.offset + dim*bytesPerDim + prefix, bytesPerDim-prefix);
}
@@ -1036,7 +1052,7 @@ final class SimpleTextBKDWriter implements Closeable {
boolean result = reader.next();
assert result;
System.arraycopy(reader.packedValue(), splitDim*bytesPerDim, scratch1, 0, bytesPerDim);
- if (numDims > 1) {
+ if (numDataDims > 1) {
assert ordBitSet.get(reader.ord()) == false;
ordBitSet.set(reader.ord());
// Subtract 1 from rightCount because we already did the first value above (so we could record the split value):
@@ -1051,7 +1067,7 @@ final class SimpleTextBKDWriter implements Closeable {
/** Called only in assert */
private boolean valueInBounds(BytesRef packedValue, byte[] minPackedValue, byte[] maxPackedValue) {
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
int offset = bytesPerDim*dim;
if (FutureArrays.compareUnsigned(packedValue.bytes, packedValue.offset + offset, packedValue.offset + offset + bytesPerDim, minPackedValue, offset, offset + bytesPerDim) < 0) {
return false;
@@ -1067,7 +1083,7 @@ final class SimpleTextBKDWriter implements Closeable {
protected int split(byte[] minPackedValue, byte[] maxPackedValue) {
// Find which dim has the largest span so we can split on it:
int splitDim = -1;
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numIndexDims;dim++) {
NumericUtils.subtract(bytesPerDim, dim, maxPackedValue, minPackedValue, scratchDiff);
if (splitDim == -1 || FutureArrays.compareUnsigned(scratchDiff, 0, bytesPerDim, scratch1, 0, bytesPerDim) > 0) {
System.arraycopy(scratchDiff, 0, scratch1, 0, bytesPerDim);
@@ -1115,7 +1131,7 @@ final class SimpleTextBKDWriter implements Closeable {
reader.getValue(from, scratchBytesRef1);
for (int i = from + 1; i < to; ++i) {
reader.getValue(i, scratchBytesRef2);
- for (int dim=0;dim<numDims;dim++) {
+ for (int dim=0;dim<numDataDims;dim++) {
final int offset = dim * bytesPerDim;
for(int j=0;j<commonPrefixLengths[dim];j++) {
if (scratchBytesRef1.bytes[scratchBytesRef1.offset+offset+j] != scratchBytesRef2.bytes[scratchBytesRef2.offset+offset+j]) {
@@ -1127,14 +1143,14 @@ final class SimpleTextBKDWriter implements Closeable {
}
// Find the dimension that has the least number of unique bytes at commonPrefixLengths[dim]
- FixedBitSet[] usedBytes = new FixedBitSet[numDims];
- for (int dim = 0; dim < numDims; ++dim) {
+ FixedBitSet[] usedBytes = new FixedBitSet[numDataDims];
+ for (int dim = 0; dim < numDataDims; ++dim) {
if (commonPrefixLengths[dim] < bytesPerDim) {
usedBytes[dim] = new FixedBitSet(256);
}
}
for (int i = from + 1; i < to; ++i) {
- for (int dim=0;dim<numDims;dim++) {
+ for (int dim=0;dim<numDataDims;dim++) {
if (usedBytes[dim] != null) {
byte b = reader.getByteAt(i, dim * bytesPerDim + commonPrefixLengths[dim]);
usedBytes[dim].set(Byte.toUnsignedInt(b));
@@ -1143,7 +1159,7 @@ final class SimpleTextBKDWriter implements Closeable {
}
int sortedDim = 0;
int sortedDimCardinality = Integer.MAX_VALUE;
- for (int dim = 0; dim < numDims; ++dim) {
+ for (int dim = 0; dim < numDataDims; ++dim) {
if (usedBytes[dim] != null) {
final int cardinality = usedBytes[dim].cardinality();
if (cardinality < sortedDimCardinality) {
@@ -1206,8 +1222,8 @@ final class SimpleTextBKDWriter implements Closeable {
reader.getValue(mid, scratchBytesRef1);
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + splitDim * bytesPerDim, splitPackedValues, address + 1, bytesPerDim);
- byte[] minSplitPackedValue = ArrayUtil.copyOfSubArray(minPackedValue, 0, packedBytesLength);
- byte[] maxSplitPackedValue = ArrayUtil.copyOfSubArray(maxPackedValue, 0, packedBytesLength);
+ byte[] minSplitPackedValue = ArrayUtil.copyOfSubArray(minPackedValue, 0, packedIndexBytesLength);
+ byte[] maxSplitPackedValue = ArrayUtil.copyOfSubArray(maxPackedValue, 0, packedIndexBytesLength);
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + splitDim * bytesPerDim,
minSplitPackedValue, splitDim * bytesPerDim, bytesPerDim);
System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + splitDim * bytesPerDim,
@@ -1235,7 +1251,7 @@ final class SimpleTextBKDWriter implements Closeable {
assert slice.count == slices[0].count;
}
- if (numDims == 1 && slices[0].writer instanceof OfflinePointWriter && slices[0].count <= maxPointsSortInHeap) {
+ if (numDataDims == 1 && slices[0].writer instanceof OfflinePointWriter && slices[0].count <= maxPointsSortInHeap) {
// Special case for 1D, to cutover to heap once we recurse deeply enough:
slices[0] = switchToHeap(slices[0], toCloseHeroically);
}
@@ -1248,7 +1264,7 @@ final class SimpleTextBKDWriter implements Closeable {
int sortedDim = 0;
int sortedDimCardinality = Integer.MAX_VALUE;
- for (int dim=0;dim<numDims;dim++) {
+ for (int dim=0;dim<numDataDims;dim++) {
if (slices[dim].writer instanceof HeapPointWriter == false) {
// Adversarial cases can cause this, e.g. very lopsided data, all equal points, such that we started
// offline, but then kept splitting only in one dimension, and so never had to rewrite into heap writer
@@ -1333,7 +1349,7 @@ final class SimpleTextBKDWriter implements Closeable {
// Inner node: partition/recurse
int splitDim;
- if (numDims > 1) {
+ if (numIndexDims > 1) {
splitDim = split(minPackedValue, maxPackedValue);
} else {
splitDim = 0;
@@ -1354,24 +1370,24 @@ final class SimpleTextBKDWriter implements Closeable {
// Partition all PathSlice that are not the split dim into sorted left and right sets, so we can recurse:
- PathSlice[] leftSlices = new PathSlice[numDims];
- PathSlice[] rightSlices = new PathSlice[numDims];
+ PathSlice[] leftSlices = new PathSlice[numDataDims];
+ PathSlice[] rightSlices = new PathSlice[numDataDims];
- byte[] minSplitPackedValue = new byte[packedBytesLength];
- System.arraycopy(minPackedValue, 0, minSplitPackedValue, 0, packedBytesLength);
+ byte[] minSplitPackedValue = new byte[packedIndexBytesLength];
+ System.arraycopy(minPackedValue, 0, minSplitPackedValue, 0, packedIndexBytesLength);
- byte[] maxSplitPackedValue = new byte[packedBytesLength];
- System.arraycopy(maxPackedValue, 0, maxSplitPackedValue, 0, packedBytesLength);
+ byte[] maxSplitPackedValue = new byte[packedIndexBytesLength];
+ System.arraycopy(maxPackedValue, 0, maxSplitPackedValue, 0, packedIndexBytesLength);
// When we are on this dim, below, we clear the ordBitSet:
int dimToClear;
- if (numDims - 1 == splitDim) {
- dimToClear = numDims - 2;
+ if (numDataDims - 1 == splitDim) {
+ dimToClear = numDataDims - 2;
} else {
- dimToClear = numDims - 1;
+ dimToClear = numDataDims - 1;
}
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
if (dim == splitDim) {
// No need to partition on this dim since it's a simple slice of the incoming already sorted slice, and we
@@ -1392,7 +1408,7 @@ final class SimpleTextBKDWriter implements Closeable {
long nextRightCount = reader.split(source.count, ordBitSet, leftPointWriter, rightPointWriter, dim == dimToClear);
if (rightCount != nextRightCount) {
- throw new IllegalStateException("wrong number of points in split: expected=" + rightCount + " but actual=" + nextRightCount);
+ throw new IllegalStateException("wrong number of points in split: expected=" + rightCount + " but actual=" + nextRightCount + " in dim " + dim);
}
leftSlices[dim] = new PathSlice(leftPointWriter, 0, leftCount);
@@ -1407,7 +1423,7 @@ final class SimpleTextBKDWriter implements Closeable {
ordBitSet, out,
minPackedValue, maxSplitPackedValue,
splitPackedValues, leafBlockFPs, toCloseHeroically);
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
// Don't destroy the dim we split on because we just re-used what our caller above gave us for that dim:
if (dim != splitDim) {
leftSlices[dim].writer.destroy();
@@ -1420,7 +1436,7 @@ final class SimpleTextBKDWriter implements Closeable {
ordBitSet, out,
minSplitPackedValue, maxPackedValue,
splitPackedValues, leafBlockFPs, toCloseHeroically);
- for(int dim=0;dim<numDims;dim++) {
+ for(int dim=0;dim<numDataDims;dim++) {
// Don't destroy the dim we split on because we just re-used what our caller above gave us for that dim:
if (dim != splitDim) {
rightSlices[dim].writer.destroy();
@@ -1454,10 +1470,10 @@ final class SimpleTextBKDWriter implements Closeable {
if (ord > 0) {
int cmp = FutureArrays.compareUnsigned(lastPackedValue, dimOffset, dimOffset + bytesPerDim, packedValue, packedValueOffset + dimOffset, packedValueOffset + dimOffset + bytesPerDim);
if (cmp > 0) {
- throw new AssertionError("values out of order: last value=" + new BytesRef(lastPackedValue) + " current value=" + new BytesRef(packedValue, packedValueOffset, packedBytesLength) + " ord=" + ord);
+ throw new AssertionError("values out of order: last value=" + new BytesRef(lastPackedValue) + " current value=" + new BytesRef(packedValue, packedValueOffset, packedBytesLength) + " ord=" + ord + " sortedDim=" + sortedDim);
}
if (cmp == 0 && doc < lastDoc) {
- throw new AssertionError("docs out of order: last doc=" + lastDoc + " current doc=" + doc + " ord=" + ord);
+ throw new AssertionError("docs out of order: last doc=" + lastDoc + " current doc=" + doc + " ord=" + ord + " sortedDim=" + sortedDim);
}
}
System.arraycopy(packedValue, packedValueOffset, lastPackedValue, 0, packedBytesLength);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
index 1c40cbd..f52a84a 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
@@ -64,7 +64,8 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
static final BytesRef NUM_ATTS = new BytesRef(" attributes ");
static final BytesRef ATT_KEY = new BytesRef(" key ");
static final BytesRef ATT_VALUE = new BytesRef(" value ");
- static final BytesRef DIM_COUNT = new BytesRef(" dimensional count ");
+ static final BytesRef DATA_DIM_COUNT = new BytesRef(" data dimensional count ");
+ static final BytesRef INDEX_DIM_COUNT = new BytesRef(" index dimensional count ");
static final BytesRef DIM_NUM_BYTES = new BytesRef(" dimensional num bytes ");
static final BytesRef SOFT_DELETES = new BytesRef(" soft-deletes ");
@@ -134,8 +135,12 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
}
SimpleTextUtil.readLine(input, scratch);
- assert StringHelper.startsWith(scratch.get(), DIM_COUNT);
- int dimensionalCount = Integer.parseInt(readString(DIM_COUNT.length, scratch));
+ assert StringHelper.startsWith(scratch.get(), DATA_DIM_COUNT);
+ int dataDimensionalCount = Integer.parseInt(readString(DATA_DIM_COUNT.length, scratch));
+
+ SimpleTextUtil.readLine(input, scratch);
+ assert StringHelper.startsWith(scratch.get(), INDEX_DIM_COUNT);
+ int indexDimensionalCount = Integer.parseInt(readString(INDEX_DIM_COUNT.length, scratch));
SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), DIM_NUM_BYTES);
@@ -147,7 +152,7 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
infos[i] = new FieldInfo(name, fieldNumber, storeTermVector,
omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(atts),
- dimensionalCount, dimensionalNumBytes, isSoftDeletesField);
+ dataDimensionalCount, indexDimensionalCount, dimensionalNumBytes, isSoftDeletesField);
}
SimpleTextUtil.checkFooter(input);
@@ -236,8 +241,12 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
}
}
- SimpleTextUtil.write(out, DIM_COUNT);
- SimpleTextUtil.write(out, Integer.toString(fi.getPointDimensionCount()), scratch);
+ SimpleTextUtil.write(out, DATA_DIM_COUNT);
+ SimpleTextUtil.write(out, Integer.toString(fi.getPointDataDimensionCount()), scratch);
+ SimpleTextUtil.writeNewline(out);
+
+ SimpleTextUtil.write(out, INDEX_DIM_COUNT);
+ SimpleTextUtil.write(out, Integer.toString(fi.getPointIndexDimensionCount()), scratch);
SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, DIM_NUM_BYTES);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsReader.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsReader.java
index 453bd23..728aad1 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsReader.java
@@ -47,7 +47,8 @@ import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.INDEX_C
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.MAX_LEAF_POINTS;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.MAX_VALUE;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.MIN_VALUE;
-import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.NUM_DIMS;
+import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.NUM_DATA_DIMS;
+import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.NUM_INDEX_DIMS;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.POINT_COUNT;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.SPLIT_COUNT;
import static org.apache.lucene.codecs.simpletext.SimpleTextPointsWriter.SPLIT_DIM;
@@ -101,7 +102,10 @@ class SimpleTextPointsReader extends PointsReader {
// NOTE: matches what writeIndex does in SimpleTextPointsWriter
dataIn.seek(fp);
readLine(dataIn);
- int numDims = parseInt(NUM_DIMS);
+ int numDataDims = parseInt(NUM_DATA_DIMS);
+
+ readLine(dataIn);
+ int numIndexDims = parseInt(NUM_INDEX_DIMS);
readLine(dataIn);
int bytesPerDim = parseInt(BYTES_PER_DIM);
@@ -115,12 +119,12 @@ class SimpleTextPointsReader extends PointsReader {
readLine(dataIn);
assert startsWith(MIN_VALUE);
BytesRef minValue = SimpleTextUtil.fromBytesRefString(stripPrefix(MIN_VALUE));
- assert minValue.length == numDims*bytesPerDim;
+ assert minValue.length == numIndexDims*bytesPerDim;
readLine(dataIn);
assert startsWith(MAX_VALUE);
BytesRef maxValue = SimpleTextUtil.fromBytesRefString(stripPrefix(MAX_VALUE));
- assert maxValue.length == numDims*bytesPerDim;
+ assert maxValue.length == numIndexDims*bytesPerDim;
readLine(dataIn);
assert startsWith(POINT_COUNT);
@@ -140,7 +144,7 @@ class SimpleTextPointsReader extends PointsReader {
byte[] splitPackedValues;
int bytesPerIndexEntry;
- if (numDims == 1) {
+ if (numIndexDims == 1) {
bytesPerIndexEntry = bytesPerDim;
} else {
bytesPerIndexEntry = 1 + bytesPerDim;
@@ -150,7 +154,7 @@ class SimpleTextPointsReader extends PointsReader {
readLine(dataIn);
int address = bytesPerIndexEntry * i;
int splitDim = parseInt(SPLIT_DIM);
- if (numDims != 1) {
+ if (numIndexDims != 1) {
splitPackedValues[address++] = (byte) splitDim;
}
readLine(dataIn);
@@ -160,7 +164,7 @@ class SimpleTextPointsReader extends PointsReader {
System.arraycopy(br.bytes, br.offset, splitPackedValues, address, bytesPerDim);
}
- return new SimpleTextBKDReader(dataIn, numDims, maxPointsInLeafNode, bytesPerDim, leafBlockFPs, splitPackedValues, minValue.bytes, maxValue.bytes, pointCount, docCount);
+ return new SimpleTextBKDReader(dataIn, numDataDims, numIndexDims, maxPointsInLeafNode, bytesPerDim, leafBlockFPs, splitPackedValues, minValue.bytes, maxValue.bytes, pointCount, docCount);
}
private void readLine(IndexInput in) throws IOException {
@@ -191,7 +195,7 @@ class SimpleTextPointsReader extends PointsReader {
if (fieldInfo == null) {
throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized");
}
- if (fieldInfo.getPointDimensionCount() == 0) {
+ if (fieldInfo.getPointDataDimensionCount() == 0) {
throw new IllegalArgumentException("field=\"" + fieldName + "\" did not index points");
}
return readers.get(fieldName);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java
index c3217f3..2da74d6 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java
@@ -35,7 +35,8 @@ import org.apache.lucene.util.BytesRefBuilder;
class SimpleTextPointsWriter extends PointsWriter {
- public final static BytesRef NUM_DIMS = new BytesRef("num dims ");
+ public final static BytesRef NUM_DATA_DIMS = new BytesRef("num data dims ");
+ public final static BytesRef NUM_INDEX_DIMS = new BytesRef("num index dims ");
public final static BytesRef BYTES_PER_DIM = new BytesRef("bytes per dim ");
public final static BytesRef MAX_LEAF_POINTS = new BytesRef("max leaf points ");
public final static BytesRef INDEX_COUNT = new BytesRef("index count ");
@@ -76,7 +77,8 @@ class SimpleTextPointsWriter extends PointsWriter {
try (SimpleTextBKDWriter writer = new SimpleTextBKDWriter(writeState.segmentInfo.maxDoc(),
writeState.directory,
writeState.segmentInfo.name,
- fieldInfo.getPointDimensionCount(),
+ fieldInfo.getPointDataDimensionCount(),
+ fieldInfo.getPointIndexDimensionCount(),
fieldInfo.getPointNumBytes(),
SimpleTextBKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE,
SimpleTextBKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
index d9a0b30..dd1eaeb 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
@@ -48,7 +48,7 @@ public abstract class PointsWriter implements Closeable {
PointsReader pointsReader = mergeState.pointsReaders[i];
if (pointsReader != null) {
FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(fieldInfo.name);
- if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
+ if (readerFieldInfo != null && readerFieldInfo.getPointDataDimensionCount() > 0) {
PointValues values = pointsReader.getValues(fieldInfo.name);
if (values != null) {
maxPointCount += values.size();
@@ -92,7 +92,7 @@ public abstract class PointsWriter implements Closeable {
continue;
}
- if (readerFieldInfo.getPointDimensionCount() == 0) {
+ if (readerFieldInfo.getPointDataDimensionCount() == 0) {
// This segment saw this field, but the field did not index points in it:
continue;
}
@@ -143,7 +143,12 @@ public abstract class PointsWriter implements Closeable {
}
@Override
- public int getNumDimensions() {
+ public int getNumDataDimensions() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getNumIndexDimensions() {
throw new UnsupportedOperationException();
}
@@ -182,7 +187,7 @@ public abstract class PointsWriter implements Closeable {
}
// merge field at a time
for (FieldInfo fieldInfo : mergeState.mergeFieldInfos) {
- if (fieldInfo.getPointDimensionCount() != 0) {
+ if (fieldInfo.getPointDataDimensionCount() != 0) {
mergeOneField(mergeState, fieldInfo);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
index 30dca70..0ad0cad 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
@@ -148,7 +148,7 @@ public final class Lucene50FieldInfosFormat extends FieldInfosFormat {
lastAttributes = attributes;
try {
infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads,
- indexOptions, docValuesType, dvGen, attributes, 0, 0, false);
+ indexOptions, docValuesType, dvGen, attributes, 0, 0, 0, false);
infos[i].checkConsistency();
} catch (IllegalStateException e) {
throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java
index 522a73f..e2ca9ee 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60FieldInfosFormat.java
@@ -114,7 +114,7 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
Throwable priorE = null;
FieldInfo infos[] = null;
try {
- CodecUtil.checkIndexHeader(input,
+ int version = CodecUtil.checkIndexHeader(input,
Lucene60FieldInfosFormat.CODEC_NAME,
Lucene60FieldInfosFormat.FORMAT_START,
Lucene60FieldInfosFormat.FORMAT_CURRENT,
@@ -149,9 +149,13 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
attributes = lastAttributes;
}
lastAttributes = attributes;
- int pointDimensionCount = input.readVInt();
+ int pointDataDimensionCount = input.readVInt();
int pointNumBytes;
- if (pointDimensionCount != 0) {
+ int pointIndexDimensionCount = pointDataDimensionCount;
+ if (pointDataDimensionCount != 0) {
+ if (version >= Lucene60FieldInfosFormat.FORMAT_SELECTIVE_INDEXING) {
+ pointIndexDimensionCount = input.readVInt();
+ }
pointNumBytes = input.readVInt();
} else {
pointNumBytes = 0;
@@ -160,7 +164,7 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
try {
infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads,
indexOptions, docValuesType, dvGen, attributes,
- pointDimensionCount, pointNumBytes, isSoftDeletesField);
+ pointDataDimensionCount, pointIndexDimensionCount, pointNumBytes, isSoftDeletesField);
infos[i].checkConsistency();
} catch (IllegalStateException e) {
throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
@@ -287,9 +291,9 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
output.writeByte(docValuesByte(fi.getDocValuesType()));
output.writeLong(fi.getDocValuesGen());
output.writeMapOfStrings(fi.attributes());
- int pointDimensionCount = fi.getPointDimensionCount();
- output.writeVInt(pointDimensionCount);
- if (pointDimensionCount != 0) {
+ output.writeVInt(fi.getPointDataDimensionCount());
+ if (fi.getPointDataDimensionCount() != 0) {
+ output.writeVInt(fi.getPointIndexDimensionCount());
output.writeVInt(fi.getPointNumBytes());
}
}
@@ -304,7 +308,8 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
static final String CODEC_NAME = "Lucene60FieldInfos";
static final int FORMAT_START = 0;
static final int FORMAT_SOFT_DELETES = 1;
- static final int FORMAT_CURRENT = FORMAT_SOFT_DELETES;
+ static final int FORMAT_SELECTIVE_INDEXING = 2;
+ static final int FORMAT_CURRENT = FORMAT_SELECTIVE_INDEXING;
// Field flags
static final byte STORE_TERMVECTOR = 0x1;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsReader.java
index 0eecdbb..b5ff680 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsReader.java
@@ -124,7 +124,7 @@ public class Lucene60PointsReader extends PointsReader implements Closeable {
if (fieldInfo == null) {
throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized");
}
- if (fieldInfo.getPointDimensionCount() == 0) {
+ if (fieldInfo.getPointDataDimensionCount() == 0) {
throw new IllegalArgumentException("field=\"" + fieldName + "\" did not index point values");
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java
index 4f51e26..fddf08c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java
@@ -94,7 +94,8 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
try (BKDWriter writer = new BKDWriter(writeState.segmentInfo.maxDoc(),
writeState.directory,
writeState.segmentInfo.name,
- fieldInfo.getPointDimensionCount(),
+ fieldInfo.getPointDataDimensionCount(),
+ fieldInfo.getPointIndexDimensionCount(),
fieldInfo.getPointNumBytes(),
maxPointsInLeafNode,
maxMBSortInHeap,
@@ -152,8 +153,8 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
}
for (FieldInfo fieldInfo : mergeState.mergeFieldInfos) {
- if (fieldInfo.getPointDimensionCount() != 0) {
- if (fieldInfo.getPointDimensionCount() == 1) {
+ if (fieldInfo.getPointDataDimensionCount() != 0) {
+ if (fieldInfo.getPointDataDimensionCount() == 1) {
boolean singleValuePerDoc = true;
@@ -164,7 +165,7 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
if (reader != null) {
FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
- if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
+ if (readerFieldInfo != null && readerFieldInfo.getPointDataDimensionCount() > 0) {
PointValues values = reader.getValues(fieldInfo.name);
if (values != null) {
totMaxSize += values.size();
@@ -181,7 +182,8 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
try (BKDWriter writer = new BKDWriter(writeState.segmentInfo.maxDoc(),
writeState.directory,
writeState.segmentInfo.name,
- fieldInfo.getPointDimensionCount(),
+ fieldInfo.getPointDataDimensionCount(),
+ fieldInfo.getPointIndexDimensionCount(),
fieldInfo.getPointNumBytes(),
maxPointsInLeafNode,
maxMBSortInHeap,
@@ -204,7 +206,7 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
- if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
+ if (readerFieldInfo != null && readerFieldInfo.getPointDataDimensionCount() > 0) {
BKDReader bkdReader = reader60.readers.get(readerFieldInfo.number);
if (bkdReader != null) {
bkdReaders.add(bkdReader);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.java
index 991eedf..291a384 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.java
@@ -134,7 +134,7 @@ final class PerFieldMergeState {
hasNorms |= fi.hasNorms();
hasDocValues |= fi.getDocValuesType() != DocValuesType.NONE;
hasPayloads |= fi.hasPayloads();
- hasPointValues |= (fi.getPointDimensionCount() != 0);
+ hasPointValues |= (fi.getPointDataDimensionCount() != 0);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java b/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java
index 693a00f..f4b625e 100644
--- a/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java
+++ b/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java
@@ -124,8 +124,8 @@ public final class BinaryPoint extends Field {
/** Expert API */
public BinaryPoint(String name, byte[] packedPoint, IndexableFieldType type) {
super(name, packedPoint, type);
- if (packedPoint.length != type.pointDimensionCount() * type.pointNumBytes()) {
- throw new IllegalArgumentException("packedPoint is length=" + packedPoint.length + " but type.pointDimensionCount()=" + type.pointDimensionCount() + " and type.pointNumBytes()=" + type.pointNumBytes());
+ if (packedPoint.length != type.pointDataDimensionCount() * type.pointNumBytes()) {
+ throw new IllegalArgumentException("packedPoint is length=" + packedPoint.length + " but type.pointDimensionCount()=" + type.pointDataDimensionCount() + " and type.pointNumBytes()=" + type.pointNumBytes());
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java
index 6547402..e442bec 100644
--- a/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java
+++ b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java
@@ -85,8 +85,8 @@ public final class DoublePoint extends Field {
/** Change the values of this field */
public void setDoubleValues(double... point) {
- if (type.pointDimensionCount() != point.length) {
- throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
+ if (type.pointDataDimensionCount() != point.length) {
+ throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
}
fieldsData = pack(point);
}
@@ -98,8 +98,8 @@ public final class DoublePoint extends Field {
@Override
public Number numericValue() {
- if (type.pointDimensionCount() != 1) {
- throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value");
+ if (type.pointDataDimensionCount() != 1) {
+ throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot convert to a single numeric value");
}
BytesRef bytes = (BytesRef) fieldsData;
assert bytes.length == Double.BYTES;
@@ -142,7 +142,7 @@ public final class DoublePoint extends Field {
result.append(':');
BytesRef bytes = (BytesRef) fieldsData;
- for (int dim = 0; dim < type.pointDimensionCount(); dim++) {
+ for (int dim = 0; dim < type.pointDataDimensionCount(); dim++) {
if (dim > 0) {
result.append(',');
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/document/DoubleRange.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/DoubleRange.java b/lucene/core/src/java/org/apache/lucene/document/DoubleRange.java
index cf308c3..726d0a1 100644
--- a/lucene/core/src/java/org/apache/lucene/document/DoubleRange.java
+++ b/lucene/core/src/java/org/apache/lucene/document/DoubleRange.java
@@ -77,8 +77,8 @@ public class DoubleRange extends Field {
*/
public void setRangeValues(double[] min, double[] max) {
checkArgs(min, max);
- if (min.length*2 != type.pointDimensionCount() || max.length*2 != type.pointDimensionCount()) {
- throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDimensionCount()/2
+ if (min.length*2 != type.pointDataDimensionCount() || max.length*2 != type.pointDataDimensionCount()) {
+ throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDataDimensionCount()/2
+ " dimensions; cannot change to (incoming) " + min.length + " dimensions");
}
@@ -148,7 +148,7 @@ public class DoubleRange extends Field {
* @return the decoded min value
*/
public double getMin(int dimension) {
- FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
+ FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMin(((BytesRef)fieldsData).bytes, dimension);
}
@@ -158,7 +158,7 @@ public class DoubleRange extends Field {
* @return the decoded max value
*/
public double getMax(int dimension) {
- FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
+ FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMax(((BytesRef)fieldsData).bytes, dimension);
}
@@ -244,7 +244,7 @@ public class DoubleRange extends Field {
sb.append(':');
byte[] b = ((BytesRef)fieldsData).bytes;
toString(b, 0);
- for (int d = 0; d < type.pointDimensionCount() / 2; ++d) {
+ for (int d = 0; d < type.pointDataDimensionCount() / 2; ++d) {
sb.append(' ');
sb.append(toString(b, d));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/document/FieldType.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/FieldType.java b/lucene/core/src/java/org/apache/lucene/document/FieldType.java
index 3c7d276..a21572e 100644
--- a/lucene/core/src/java/org/apache/lucene/document/FieldType.java
+++ b/lucene/core/src/java/org/apache/lucene/document/FieldType.java
@@ -38,7 +38,8 @@ public class FieldType implements IndexableFieldType {
private IndexOptions indexOptions = IndexOptions.NONE;
private boolean frozen;
private DocValuesType docValuesType = DocValuesType.NONE;
- private int dimensionCount;
+ private int dataDimensionCount;
+ private int indexDimensionCount;
private int dimensionNumBytes;
/**
@@ -54,7 +55,8 @@ public class FieldType implements IndexableFieldType {
this.omitNorms = ref.omitNorms();
this.indexOptions = ref.indexOptions();
this.docValuesType = ref.docValuesType();
- this.dimensionCount = ref.pointDimensionCount();
+ this.dataDimensionCount = ref.pointDataDimensionCount();
+ this.indexDimensionCount = ref.pointIndexDimensionCount();
this.dimensionNumBytes = ref.pointNumBytes();
// Do not copy frozen!
}
@@ -279,11 +281,24 @@ public class FieldType implements IndexableFieldType {
* Enables points indexing.
*/
public void setDimensions(int dimensionCount, int dimensionNumBytes) {
- if (dimensionCount < 0) {
- throw new IllegalArgumentException("dimensionCount must be >= 0; got " + dimensionCount);
+ this.setDimensions(dimensionCount, dimensionCount, dimensionNumBytes);
+ }
+
+ /**
+ * Enables points indexing with selectable dimension indexing.
+ */
+ public void setDimensions(int dataDimensionCount, int indexDimensionCount, int dimensionNumBytes) {
+ if (dataDimensionCount < 0) {
+ throw new IllegalArgumentException("dataDimensionCount must be >= 0; got " + dataDimensionCount);
+ }
+ if (dataDimensionCount > PointValues.MAX_DIMENSIONS) {
+ throw new IllegalArgumentException("dataDimensionCount must be <= " + PointValues.MAX_DIMENSIONS + "; got " + dataDimensionCount);
}
- if (dimensionCount > PointValues.MAX_DIMENSIONS) {
- throw new IllegalArgumentException("dimensionCount must be <= " + PointValues.MAX_DIMENSIONS + "; got " + dimensionCount);
+ if (indexDimensionCount < 0) {
+ throw new IllegalArgumentException("indexDimensionCount must be >= 0; got " + indexDimensionCount);
+ }
+ if (indexDimensionCount > dataDimensionCount) {
+ throw new IllegalArgumentException("indexDimensionCount must be <= dataDimensionCount: " + dataDimensionCount + "; got " + indexDimensionCount);
}
if (dimensionNumBytes < 0) {
throw new IllegalArgumentException("dimensionNumBytes must be >= 0; got " + dimensionNumBytes);
@@ -291,23 +306,34 @@ public class FieldType implements IndexableFieldType {
if (dimensionNumBytes > PointValues.MAX_NUM_BYTES) {
throw new IllegalArgumentException("dimensionNumBytes must be <= " + PointValues.MAX_NUM_BYTES + "; got " + dimensionNumBytes);
}
- if (dimensionCount == 0) {
+ if (dataDimensionCount == 0) {
+ if (indexDimensionCount != 0) {
+ throw new IllegalArgumentException("when dataDimensionCount is 0, indexDimensionCount must be 0; got " + indexDimensionCount);
+ }
if (dimensionNumBytes != 0) {
- throw new IllegalArgumentException("when dimensionCount is 0, dimensionNumBytes must 0; got " + dimensionNumBytes);
+ throw new IllegalArgumentException("when dataDimensionCount is 0, dimensionNumBytes must be 0; got " + dimensionNumBytes);
}
+ } else if (indexDimensionCount == 0) {
+ throw new IllegalArgumentException("when dataDimensionCount is > 0, indexDimensionCount must be > 0; got " + indexDimensionCount);
} else if (dimensionNumBytes == 0) {
- if (dimensionCount != 0) {
- throw new IllegalArgumentException("when dimensionNumBytes is 0, dimensionCount must 0; got " + dimensionCount);
+ if (dataDimensionCount != 0) {
+ throw new IllegalArgumentException("when dimensionNumBytes is 0, dataDimensionCount must be 0; got " + dataDimensionCount);
}
}
- this.dimensionCount = dimensionCount;
+ this.dataDimensionCount = dataDimensionCount;
+ this.indexDimensionCount = indexDimensionCount;
this.dimensionNumBytes = dimensionNumBytes;
}
@Override
- public int pointDimensionCount() {
- return dimensionCount;
+ public int pointDataDimensionCount() {
+ return dataDimensionCount;
+ }
+
+ @Override
+ public int pointIndexDimensionCount() {
+ return indexDimensionCount;
}
@Override
@@ -349,12 +375,14 @@ public class FieldType implements IndexableFieldType {
result.append(indexOptions);
}
}
- if (dimensionCount != 0) {
+ if (dataDimensionCount != 0) {
if (result.length() > 0) {
result.append(",");
}
- result.append("pointDimensionCount=");
- result.append(dimensionCount);
+ result.append("pointDataDimensionCount=");
+ result.append(dataDimensionCount);
+ result.append(",pointIndexDimensionCount=");
+ result.append(indexDimensionCount);
result.append(",pointNumBytes=");
result.append(dimensionNumBytes);
}
@@ -399,7 +427,8 @@ public class FieldType implements IndexableFieldType {
public int hashCode() {
final int prime = 31;
int result = 1;
- result = prime * result + dimensionCount;
+ result = prime * result + dataDimensionCount;
+ result = prime * result + indexDimensionCount;
result = prime * result + dimensionNumBytes;
result = prime * result + ((docValuesType == null) ? 0 : docValuesType.hashCode());
result = prime * result + indexOptions.hashCode();
@@ -419,7 +448,8 @@ public class FieldType implements IndexableFieldType {
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
FieldType other = (FieldType) obj;
- if (dimensionCount != other.dimensionCount) return false;
+ if (dataDimensionCount != other.dataDimensionCount) return false;
+ if (indexDimensionCount != other.indexDimensionCount) return false;
if (dimensionNumBytes != other.dimensionNumBytes) return false;
if (docValuesType != other.docValuesType) return false;
if (indexOptions != other.indexOptions) return false;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java
index 0ec67fd..b6d508f 100644
--- a/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java
+++ b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java
@@ -85,8 +85,8 @@ public final class FloatPoint extends Field {
/** Change the values of this field */
public void setFloatValues(float... point) {
- if (type.pointDimensionCount() != point.length) {
- throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
+ if (type.pointDataDimensionCount() != point.length) {
+ throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
}
fieldsData = pack(point);
}
@@ -98,8 +98,8 @@ public final class FloatPoint extends Field {
@Override
public Number numericValue() {
- if (type.pointDimensionCount() != 1) {
- throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value");
+ if (type.pointDataDimensionCount() != 1) {
+ throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot convert to a single numeric value");
}
BytesRef bytes = (BytesRef) fieldsData;
assert bytes.length == Float.BYTES;
@@ -142,7 +142,7 @@ public final class FloatPoint extends Field {
result.append(':');
BytesRef bytes = (BytesRef) fieldsData;
- for (int dim = 0; dim < type.pointDimensionCount(); dim++) {
+ for (int dim = 0; dim < type.pointDataDimensionCount(); dim++) {
if (dim > 0) {
result.append(',');
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/document/FloatRange.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/FloatRange.java b/lucene/core/src/java/org/apache/lucene/document/FloatRange.java
index 9b555d6..191631b 100644
--- a/lucene/core/src/java/org/apache/lucene/document/FloatRange.java
+++ b/lucene/core/src/java/org/apache/lucene/document/FloatRange.java
@@ -77,8 +77,8 @@ public class FloatRange extends Field {
*/
public void setRangeValues(float[] min, float[] max) {
checkArgs(min, max);
- if (min.length*2 != type.pointDimensionCount() || max.length*2 != type.pointDimensionCount()) {
- throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDimensionCount()/2
+ if (min.length*2 != type.pointDataDimensionCount() || max.length*2 != type.pointDataDimensionCount()) {
+ throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDataDimensionCount()/2
+ " dimensions; cannot change to (incoming) " + min.length + " dimensions");
}
@@ -148,7 +148,7 @@ public class FloatRange extends Field {
* @return the decoded min value
*/
public float getMin(int dimension) {
- FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
+ FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMin(((BytesRef)fieldsData).bytes, dimension);
}
@@ -158,7 +158,7 @@ public class FloatRange extends Field {
* @return the decoded max value
*/
public float getMax(int dimension) {
- FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
+ FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMax(((BytesRef)fieldsData).bytes, dimension);
}
@@ -244,7 +244,7 @@ public class FloatRange extends Field {
sb.append(':');
byte[] b = ((BytesRef)fieldsData).bytes;
toString(b, 0);
- for (int d = 0; d < type.pointDimensionCount() / 2; ++d) {
+ for (int d = 0; d < type.pointDataDimensionCount() / 2; ++d) {
sb.append(' ');
sb.append(toString(b, d));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/document/IntPoint.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/IntPoint.java b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java
index 8b3484b..da4f391 100644
--- a/lucene/core/src/java/org/apache/lucene/document/IntPoint.java
+++ b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java
@@ -59,8 +59,8 @@ public final class IntPoint extends Field {
/** Change the values of this field */
public void setIntValues(int... point) {
- if (type.pointDimensionCount() != point.length) {
- throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
+ if (type.pointDataDimensionCount() != point.length) {
+ throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
}
fieldsData = pack(point);
}
@@ -72,8 +72,8 @@ public final class IntPoint extends Field {
@Override
public Number numericValue() {
- if (type.pointDimensionCount() != 1) {
- throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value");
+ if (type.pointDataDimensionCount() != 1) {
+ throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot convert to a single numeric value");
}
BytesRef bytes = (BytesRef) fieldsData;
assert bytes.length == Integer.BYTES;
@@ -116,7 +116,7 @@ public final class IntPoint extends Field {
result.append(':');
BytesRef bytes = (BytesRef) fieldsData;
- for (int dim = 0; dim < type.pointDimensionCount(); dim++) {
+ for (int dim = 0; dim < type.pointDataDimensionCount(); dim++) {
if (dim > 0) {
result.append(',');
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/document/IntRange.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/IntRange.java b/lucene/core/src/java/org/apache/lucene/document/IntRange.java
index e67b94f..b24e80a 100644
--- a/lucene/core/src/java/org/apache/lucene/document/IntRange.java
+++ b/lucene/core/src/java/org/apache/lucene/document/IntRange.java
@@ -77,8 +77,8 @@ public class IntRange extends Field {
*/
public void setRangeValues(int[] min, int[] max) {
checkArgs(min, max);
- if (min.length*2 != type.pointDimensionCount() || max.length*2 != type.pointDimensionCount()) {
- throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDimensionCount()/2
+ if (min.length*2 != type.pointDataDimensionCount() || max.length*2 != type.pointDataDimensionCount()) {
+ throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDataDimensionCount()/2
+ " dimensions; cannot change to (incoming) " + min.length + " dimensions");
}
@@ -148,7 +148,7 @@ public class IntRange extends Field {
* @return the decoded min value
*/
public int getMin(int dimension) {
- FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
+ FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMin(((BytesRef)fieldsData).bytes, dimension);
}
@@ -158,7 +158,7 @@ public class IntRange extends Field {
* @return the decoded max value
*/
public int getMax(int dimension) {
- FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
+ FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMax(((BytesRef)fieldsData).bytes, dimension);
}
@@ -244,7 +244,7 @@ public class IntRange extends Field {
sb.append(':');
byte[] b = ((BytesRef)fieldsData).bytes;
toString(b, 0);
- for (int d = 0; d < type.pointDimensionCount() / 2; ++d) {
+ for (int d = 0; d < type.pointDataDimensionCount() / 2; ++d) {
sb.append(' ');
sb.append(toString(b, d));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/document/LatLonPoint.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/core/src/java/org/apache/lucene/document/LatLonPoint.java
index e3edead..c3ecc8a 100644
--- a/lucene/core/src/java/org/apache/lucene/document/LatLonPoint.java
+++ b/lucene/core/src/java/org/apache/lucene/document/LatLonPoint.java
@@ -147,9 +147,9 @@ public class LatLonPoint extends Field {
/** helper: checks a fieldinfo and throws exception if its definitely not a LatLonPoint */
static void checkCompatible(FieldInfo fieldInfo) {
// point/dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment.
- if (fieldInfo.getPointDimensionCount() != 0 && fieldInfo.getPointDimensionCount() != TYPE.pointDimensionCount()) {
- throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with numDims=" + fieldInfo.getPointDimensionCount() +
- " but this point type has numDims=" + TYPE.pointDimensionCount() +
+ if (fieldInfo.getPointDataDimensionCount() != 0 && fieldInfo.getPointDataDimensionCount() != TYPE.pointDataDimensionCount()) {
+ throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with numDims=" + fieldInfo.getPointDataDimensionCount() +
+ " but this point type has numDims=" + TYPE.pointDataDimensionCount() +
", is the field really a LatLonPoint?");
}
if (fieldInfo.getPointNumBytes() != 0 && fieldInfo.getPointNumBytes() != TYPE.pointNumBytes()) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/document/LongPoint.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/LongPoint.java b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java
index 5311114..835a37a 100644
--- a/lucene/core/src/java/org/apache/lucene/document/LongPoint.java
+++ b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java
@@ -62,8 +62,8 @@ public final class LongPoint extends Field {
/** Change the values of this field */
public void setLongValues(long... point) {
- if (type.pointDimensionCount() != point.length) {
- throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
+ if (type.pointDataDimensionCount() != point.length) {
+ throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions");
}
fieldsData = pack(point);
}
@@ -75,8 +75,8 @@ public final class LongPoint extends Field {
@Override
public Number numericValue() {
- if (type.pointDimensionCount() != 1) {
- throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value");
+ if (type.pointDataDimensionCount() != 1) {
+ throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDataDimensionCount() + " dimensions; cannot convert to a single numeric value");
}
BytesRef bytes = (BytesRef) fieldsData;
assert bytes.length == Long.BYTES;
@@ -119,7 +119,7 @@ public final class LongPoint extends Field {
result.append(':');
BytesRef bytes = (BytesRef) fieldsData;
- for (int dim = 0; dim < type.pointDimensionCount(); dim++) {
+ for (int dim = 0; dim < type.pointDataDimensionCount(); dim++) {
if (dim > 0) {
result.append(',');
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1118299c/lucene/core/src/java/org/apache/lucene/document/LongRange.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/LongRange.java b/lucene/core/src/java/org/apache/lucene/document/LongRange.java
index 1a1b19a..9056d2d 100644
--- a/lucene/core/src/java/org/apache/lucene/document/LongRange.java
+++ b/lucene/core/src/java/org/apache/lucene/document/LongRange.java
@@ -77,8 +77,8 @@ public class LongRange extends Field {
*/
public void setRangeValues(long[] min, long[] max) {
checkArgs(min, max);
- if (min.length*2 != type.pointDimensionCount() || max.length*2 != type.pointDimensionCount()) {
- throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDimensionCount()/2
+ if (min.length*2 != type.pointDataDimensionCount() || max.length*2 != type.pointDataDimensionCount()) {
+ throw new IllegalArgumentException("field (name=" + name + ") uses " + type.pointDataDimensionCount()/2
+ " dimensions; cannot change to (incoming) " + min.length + " dimensions");
}
@@ -146,7 +146,7 @@ public class LongRange extends Field {
* @return the decoded min value
*/
public long getMin(int dimension) {
- FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
+ FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMin(((BytesRef)fieldsData).bytes, dimension);
}
@@ -156,7 +156,7 @@ public class LongRange extends Field {
* @return the decoded max value
*/
public long getMax(int dimension) {
- FutureObjects.checkIndex(dimension, type.pointDimensionCount()/2);
+ FutureObjects.checkIndex(dimension, type.pointDataDimensionCount()/2);
return decodeMax(((BytesRef)fieldsData).bytes, dimension);
}
@@ -242,7 +242,7 @@ public class LongRange extends Field {
sb.append(':');
byte[] b = ((BytesRef)fieldsData).bytes;
toString(b, 0);
- for (int d = 0; d < type.pointDimensionCount() / 2; ++d) {
+ for (int d = 0; d < type.pointDataDimensionCount() / 2; ++d) {
sb.append(' ');
sb.append(toString(b, d));
}