You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ry...@apache.org on 2012/03/13 17:43:48 UTC
svn commit: r1300232 [2/14] - in
/lucene/dev/branches/lucene3795_lsp_spatial_module: ./ dev-tools/eclipse/
dev-tools/maven/ dev-tools/maven/lucene/contrib/demo/
dev-tools/maven/solr/contrib/clustering/
dev-tools/maven/solr/contrib/dataimporthandler/ de...
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedDerefBytesImpl.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedDerefBytesImpl.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedDerefBytesImpl.java Tue Mar 13 16:43:38 2012
@@ -46,7 +46,7 @@ class FixedDerefBytesImpl {
public static class Writer extends DerefBytesWriterBase {
public Writer(Directory dir, String id, Counter bytesUsed, IOContext context)
throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, Type.BYTES_FIXED_DEREF);
}
@Override
@@ -84,7 +84,7 @@ class FixedDerefBytesImpl {
@Override
public Source getDirectSource()
throws IOException {
- return new DirectFixedDerefSource(cloneData(), cloneIndex(), size, type());
+ return new DirectFixedDerefSource(cloneData(), cloneIndex(), size, getType());
}
@Override
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java Tue Mar 13 16:43:38 2012
@@ -58,7 +58,7 @@ class FixedSortedBytesImpl {
public Writer(Directory dir, String id, Comparator<BytesRef> comp,
Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, fasterButMoreRam);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, fasterButMoreRam, Type.BYTES_FIXED_SORTED);
this.comp = comp;
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedStraightBytesImpl.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedStraightBytesImpl.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedStraightBytesImpl.java Tue Mar 13 16:43:38 2012
@@ -22,10 +22,12 @@ import java.io.IOException;
import org.apache.lucene.codecs.lucene40.values.Bytes.BytesReaderBase;
import org.apache.lucene.codecs.lucene40.values.Bytes.BytesSourceBase;
import org.apache.lucene.codecs.lucene40.values.Bytes.BytesWriterBase;
+import org.apache.lucene.document.DocValuesField;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -52,6 +54,7 @@ class FixedStraightBytesImpl {
static final int VERSION_CURRENT = VERSION_START;
static abstract class FixedBytesWriterBase extends BytesWriterBase {
+ protected final DocValuesField bytesSpareField = new DocValuesField("", new BytesRef(), Type.BYTES_FIXED_STRAIGHT);
protected int lastDocID = -1;
// start at -1 if the first added value is > 0
protected int size = -1;
@@ -60,13 +63,20 @@ class FixedStraightBytesImpl {
protected FixedBytesWriterBase(Directory dir, String id, String codecName,
int version, Counter bytesUsed, IOContext context) throws IOException {
- super(dir, id, codecName, version, bytesUsed, context);
+ this(dir, id, codecName, version, bytesUsed, context, Type.BYTES_FIXED_STRAIGHT);
+ }
+
+ protected FixedBytesWriterBase(Directory dir, String id, String codecName,
+ int version, Counter bytesUsed, IOContext context, Type type) throws IOException {
+ super(dir, id, codecName, version, bytesUsed, context, type);
pool = new ByteBlockPool(new DirectTrackingAllocator(bytesUsed));
pool.nextBuffer();
}
@Override
- protected void add(int docID, BytesRef bytes) throws IOException {
+ public void add(int docID, IndexableField value) throws IOException {
+ final BytesRef bytes = value.binaryValue();
+ assert bytes != null;
assert lastDocID < docID;
if (size == -1) {
@@ -277,7 +287,7 @@ class FixedStraightBytesImpl {
@Override
public Source getDirectSource() throws IOException {
- return new DirectFixedStraightSource(cloneData(), size, type());
+ return new DirectFixedStraightSource(cloneData(), size, getType());
}
@Override
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Floats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Floats.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Floats.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Floats.java Tue Mar 13 16:43:38 2012
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.lucene4
*/
import java.io.IOException;
+import org.apache.lucene.codecs.DocValuesArraySource;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.DocValues.Type;
@@ -39,7 +40,7 @@ import org.apache.lucene.util.IOUtils;
*
* @lucene.experimental
*/
-class Floats {
+public class Floats {
protected static final String CODEC_NAME = "Floats";
protected static final int VERSION_START = 0;
@@ -69,31 +70,28 @@ class Floats {
final static class FloatsWriter extends FixedStraightBytesImpl.Writer {
private final int size;
- private final DocValuesArray template;
+ private final DocValuesArraySource template;
public FloatsWriter(Directory dir, String id, Counter bytesUsed,
IOContext context, Type type) throws IOException {
super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
size = typeToSize(type);
this.bytesRef = new BytesRef(size);
bytesRef.length = size;
- template = DocValuesArray.TEMPLATES.get(type);
+ template = DocValuesArraySource.forType(type);
assert template != null;
}
- protected void add(int docID, double v) throws IOException {
- template.toBytes(v, bytesRef);
- add(docID, bytesRef);
- }
-
@Override
- public void add(int docID, IndexableField docValue) throws IOException {
- add(docID, docValue.numericValue().doubleValue());
+ protected boolean tryBulkMerge(DocValues docValues) {
+ // only bulk merge if value type is the same otherwise size differs
+ return super.tryBulkMerge(docValues) && docValues.getType() == template.getType();
}
@Override
- protected boolean tryBulkMerge(DocValues docValues) {
- // only bulk merge if value type is the same otherwise size differs
- return super.tryBulkMerge(docValues) && docValues.type() == template.type();
+ public void add(int docID, IndexableField value) throws IOException {
+ template.toBytes(value.numericValue().doubleValue(), bytesRef);
+ bytesSpareField.setBytesValue(bytesRef);
+ super.add(docID, bytesSpareField);
}
@Override
@@ -104,11 +102,11 @@ class Floats {
}
final static class FloatsReader extends FixedStraightBytesImpl.FixedStraightReader {
- final DocValuesArray arrayTemplate;
+ final DocValuesArraySource arrayTemplate;
FloatsReader(Directory dir, String id, int maxDoc, IOContext context, Type type)
throws IOException {
super(dir, id, CODEC_NAME, VERSION_CURRENT, maxDoc, context, type);
- arrayTemplate = DocValuesArray.TEMPLATES.get(type);
+ arrayTemplate = DocValuesArraySource.forType(type);
assert size == 4 || size == 8: "wrong size=" + size + " type=" + type + " id=" + id;
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Ints.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Ints.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Ints.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Ints.java Tue Mar 13 16:43:38 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
+import org.apache.lucene.codecs.DocValuesArraySource;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.DocValues.Type;
@@ -36,7 +37,7 @@ import org.apache.lucene.util.IOUtils;
*
* @lucene.experimental
*/
-final class Ints {
+public final class Ints {
protected static final String CODEC_NAME = "Ints";
protected static final int VERSION_START = 0;
protected static final int VERSION_CURRENT = VERSION_START;
@@ -88,7 +89,7 @@ final class Ints {
static class IntsWriter extends FixedStraightBytesImpl.Writer {
- private final DocValuesArray template;
+ private final DocValuesArraySource template;
public IntsWriter(Directory dir, String id, Counter bytesUsed,
IOContext context, Type valueType) throws IOException {
@@ -101,17 +102,7 @@ final class Ints {
size = typeToSize(valueType);
this.bytesRef = new BytesRef(size);
bytesRef.length = size;
- template = DocValuesArray.TEMPLATES.get(valueType);
- }
-
- protected void add(int docID, long v) throws IOException {
- template.toBytes(v, bytesRef);
- add(docID, bytesRef);
- }
-
- @Override
- public void add(int docID, IndexableField docValue) throws IOException {
- add(docID, docValue.numericValue().longValue());
+ template = DocValuesArraySource.forType(valueType);
}
@Override
@@ -121,20 +112,27 @@ final class Ints {
}
@Override
+ public void add(int docID, IndexableField value) throws IOException {
+ template.toBytes(value.numericValue().longValue(), bytesRef);
+ bytesSpareField.setBytesValue(bytesRef);
+ super.add(docID, bytesSpareField);
+ }
+
+ @Override
protected boolean tryBulkMerge(DocValues docValues) {
// only bulk merge if value type is the same otherwise size differs
- return super.tryBulkMerge(docValues) && docValues.type() == template.type();
+ return super.tryBulkMerge(docValues) && docValues.getType() == template.getType();
}
}
final static class IntsReader extends FixedStraightBytesImpl.FixedStraightReader {
- private final DocValuesArray arrayTemplate;
+ private final DocValuesArraySource arrayTemplate;
IntsReader(Directory dir, String id, int maxDoc, IOContext context, Type type)
throws IOException {
super(dir, id, CODEC_NAME, VERSION_CURRENT, maxDoc,
context, type);
- arrayTemplate = DocValuesArray.TEMPLATES.get(type);
+ arrayTemplate = DocValuesArraySource.forType(type);
assert arrayTemplate != null;
assert type == sizeToType(size);
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/PackedIntValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/PackedIntValues.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/PackedIntValues.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/PackedIntValues.java Tue Mar 13 16:43:38 2012
@@ -18,9 +18,8 @@ package org.apache.lucene.codecs.lucene4
*/
import java.io.IOException;
-import org.apache.lucene.codecs.lucene40.values.DocValuesArray.LongValues;
+import org.apache.lucene.codecs.DocValuesArraySource;
import org.apache.lucene.codecs.lucene40.values.FixedStraightBytesImpl.FixedBytesWriterBase;
-import org.apache.lucene.document.Field;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.index.DocValues;
@@ -59,27 +58,10 @@ class PackedIntValues {
protected PackedIntsWriter(Directory dir, String id, Counter bytesUsed,
IOContext context) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, Type.VAR_INTS);
bytesRef = new BytesRef(8);
}
-
- protected void add(int docID, long v) throws IOException {
- assert lastDocId < docID;
- if (!started) {
- started = true;
- minValue = maxValue = v;
- } else {
- if (v < minValue) {
- minValue = v;
- } else if (v > maxValue) {
- maxValue = v;
- }
- }
- lastDocId = docID;
- BytesRefUtils.copyLong(bytesRef, v);
- add(docID, bytesRef);
- }
-
+
@Override
public void finish(int docCount) throws IOException {
boolean success = false;
@@ -112,13 +94,6 @@ class PackedIntValues {
}
}
- @Override
- protected void mergeDoc(Field scratchField, Source source, int docID, int sourceDoc) throws IOException {
- assert docID > lastDocId : "docID: " + docID
- + " must be greater than the last added doc id: " + lastDocId;
- add(docID, source.getInt(sourceDoc));
- }
-
private void writePackedInts(IndexOutput datOut, int docCount) throws IOException {
datOut.writeLong(minValue);
@@ -149,10 +124,25 @@ class PackedIntValues {
}
w.finish();
}
-
+
@Override
public void add(int docID, IndexableField docValue) throws IOException {
- add(docID, docValue.numericValue().longValue());
+ final long v = docValue.numericValue().longValue();
+ assert lastDocId < docID;
+ if (!started) {
+ started = true;
+ minValue = maxValue = v;
+ } else {
+ if (v < minValue) {
+ minValue = v;
+ } else if (v > maxValue) {
+ maxValue = v;
+ }
+ }
+ lastDocId = docID;
+ DocValuesArraySource.copyLong(bytesRef, v);
+ bytesSpareField.setBytesValue(bytesRef);
+ super.add(docID, bytesSpareField);
}
}
@@ -164,7 +154,7 @@ class PackedIntValues {
private final IndexInput datIn;
private final byte type;
private final int numDocs;
- private final LongValues values;
+ private final DocValuesArraySource values;
protected PackedIntsReader(Directory dir, String id, int numDocs,
IOContext context) throws IOException {
@@ -176,7 +166,7 @@ class PackedIntValues {
try {
CodecUtil.checkHeader(datIn, CODEC_NAME, VERSION_START, VERSION_START);
type = datIn.readByte();
- values = type == FIXED_64 ? new LongValues() : null;
+ values = type == FIXED_64 ? DocValuesArraySource.forType(Type.FIXED_INTS_64) : null;
success = true;
} finally {
if (!success) {
@@ -220,7 +210,7 @@ class PackedIntValues {
@Override
- public Type type() {
+ public Type getType() {
return Type.VAR_INTS;
}
@@ -247,7 +237,7 @@ class PackedIntValues {
@Override
public BytesRef getBytes(int docID, BytesRef ref) {
ref.grow(8);
- BytesRefUtils.copyLong(ref, getInt(docID));
+ DocValuesArraySource.copyLong(ref, getInt(docID));
return ref;
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarDerefBytesImpl.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarDerefBytesImpl.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarDerefBytesImpl.java Tue Mar 13 16:43:38 2012
@@ -57,7 +57,7 @@ class VarDerefBytesImpl {
static class Writer extends DerefBytesWriterBase {
public Writer(Directory dir, String id, Counter bytesUsed, IOContext context)
throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, Type.BYTES_VAR_DEREF);
size = 0;
}
@@ -105,7 +105,7 @@ class VarDerefBytesImpl {
@Override
public Source getDirectSource()
throws IOException {
- return new DirectVarDerefSource(cloneData(), cloneIndex(), type());
+ return new DirectVarDerefSource(cloneData(), cloneIndex(), getType());
}
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java Tue Mar 13 16:43:38 2012
@@ -59,7 +59,7 @@ final class VarSortedBytesImpl {
public Writer(Directory dir, String id, Comparator<BytesRef> comp,
Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, fasterButMoreRam);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, fasterButMoreRam, Type.BYTES_VAR_SORTED);
this.comp = comp;
size = 0;
}
@@ -166,7 +166,7 @@ final class VarSortedBytesImpl {
@Override
public Source getDirectSource() throws IOException {
- return new DirectSortedSource(cloneData(), cloneIndex(), comparator, type());
+ return new DirectSortedSource(cloneData(), cloneIndex(), comparator, getType());
}
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java Tue Mar 13 16:43:38 2012
@@ -26,6 +26,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -63,7 +64,7 @@ class VarStraightBytesImpl {
private boolean merge = false;
public Writer(Directory dir, String id, Counter bytesUsed, IOContext context)
throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, Type.BYTES_VAR_STRAIGHT);
pool = new ByteBlockPool(new DirectTrackingAllocator(bytesUsed));
docToAddress = new long[1];
pool.nextBuffer(); // init
@@ -84,7 +85,9 @@ class VarStraightBytesImpl {
}
@Override
- protected void add(int docID, BytesRef bytes) throws IOException {
+ public void add(int docID, IndexableField value) throws IOException {
+ final BytesRef bytes = value.binaryValue();
+ assert bytes != null;
assert !merge;
if (bytes.length == 0) {
return; // default
@@ -245,7 +248,7 @@ class VarStraightBytesImpl {
@Override
public Source getDirectSource()
throws IOException {
- return new DirectVarStraightSource(cloneData(), cloneIndex(), type());
+ return new DirectVarStraightSource(cloneData(), cloneIndex(), getType());
}
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java Tue Mar 13 16:43:38 2012
@@ -40,6 +40,7 @@ import org.apache.lucene.util.Counter;
*/
abstract class Writer extends DocValuesConsumer {
protected final Counter bytesUsed;
+ protected Type type;
/**
* Creates a new {@link Writer}.
@@ -49,9 +50,19 @@ abstract class Writer extends DocValuesC
* internally allocated memory. All tracked bytes must be released
* once {@link #finish(int)} has been called.
*/
- protected Writer(Counter bytesUsed) {
+ protected Writer(Counter bytesUsed, Type type) {
this.bytesUsed = bytesUsed;
+ this.type = type;
}
+
+
+
+ @Override
+ protected Type getType() {
+ return type;
+ }
+
+
/**
* Factory method to create a {@link Writer} instance for a given type. This
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java Tue Mar 13 16:43:38 2012
@@ -98,8 +98,6 @@ public class MemoryPostingsFormat extend
return "PostingsFormat(name=" + getName() + " doPackFST= " + doPackFST + ")";
}
- private static final boolean VERBOSE = false;
-
private final static class TermsWriter extends TermsConsumer {
private final IndexOutput out;
private final FieldInfo field;
@@ -123,10 +121,13 @@ public class MemoryPostingsFormat extend
// NOTE: not private so we don't pay access check at runtime:
int docCount;
RAMOutputStream buffer = new RAMOutputStream();
+
+ int lastOffsetLength;
+ int lastOffset;
@Override
public void startDoc(int docID, int termDocFreq) throws IOException {
- if (VERBOSE) System.out.println(" startDoc docID=" + docID + " freq=" + termDocFreq);
+ //System.out.println(" startDoc docID=" + docID + " freq=" + termDocFreq);
final int delta = docID - lastDocID;
assert docID == 0 || delta > 0;
lastDocID = docID;
@@ -143,20 +144,23 @@ public class MemoryPostingsFormat extend
}
lastPos = 0;
+ lastOffset = 0;
}
@Override
public void addPosition(int pos, BytesRef payload, int startOffset, int endOffset) throws IOException {
assert payload == null || field.storePayloads;
- if (VERBOSE) System.out.println(" addPos pos=" + pos + " payload=" + payload);
+ //System.out.println(" addPos pos=" + pos + " payload=" + payload);
final int delta = pos - lastPos;
assert delta >= 0;
lastPos = pos;
+ int payloadLen = 0;
+
if (field.storePayloads) {
- final int payloadLen = payload == null ? 0 : payload.length;
+ payloadLen = payload == null ? 0 : payload.length;
if (payloadLen != lastPayloadLen) {
lastPayloadLen = payloadLen;
buffer.writeVInt((delta<<1)|1);
@@ -164,13 +168,28 @@ public class MemoryPostingsFormat extend
} else {
buffer.writeVInt(delta<<1);
}
-
- if (payloadLen > 0) {
- buffer.writeBytes(payload.bytes, payload.offset, payloadLen);
- }
} else {
buffer.writeVInt(delta);
}
+
+ if (field.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) {
+ // don't use startOffset - lastEndOffset, because this creates lots of negative vints for synonyms,
+ // and the numbers aren't that much smaller anyways.
+ int offsetDelta = startOffset - lastOffset;
+ int offsetLength = endOffset - startOffset;
+ if (offsetLength != lastOffsetLength) {
+ buffer.writeVInt(offsetDelta << 1 | 1);
+ buffer.writeVInt(offsetLength);
+ } else {
+ buffer.writeVInt(offsetDelta << 1);
+ }
+ lastOffset = startOffset;
+ lastOffsetLength = offsetLength;
+ }
+
+ if (payloadLen > 0) {
+ buffer.writeBytes(payload.bytes, payload.offset, payloadLen);
+ }
}
@Override
@@ -182,6 +201,8 @@ public class MemoryPostingsFormat extend
lastDocID = 0;
docCount = 0;
lastPayloadLen = 0;
+ // force first offset to write its length
+ lastOffsetLength = -1;
return this;
}
}
@@ -190,7 +211,7 @@ public class MemoryPostingsFormat extend
@Override
public PostingsConsumer startTerm(BytesRef text) {
- if (VERBOSE) System.out.println(" startTerm term=" + text.utf8ToString());
+ //System.out.println(" startTerm term=" + text.utf8ToString());
return postingsWriter.reset();
}
@@ -224,12 +245,12 @@ public class MemoryPostingsFormat extend
spare.bytes = finalBuffer;
spare.length = totalBytes;
- if (VERBOSE) {
- System.out.println(" finishTerm term=" + text.utf8ToString() + " " + totalBytes + " bytes totalTF=" + stats.totalTermFreq);
- for(int i=0;i<totalBytes;i++) {
- System.out.println(" " + Integer.toHexString(finalBuffer[i]&0xFF));
- }
- }
+
+ //System.out.println(" finishTerm term=" + text.utf8ToString() + " " + totalBytes + " bytes totalTF=" + stats.totalTermFreq);
+ //for(int i=0;i<totalBytes;i++) {
+ // System.out.println(" " + Integer.toHexString(finalBuffer[i]&0xFF));
+ //}
+
builder.add(Util.toIntsRef(text, scratchIntsRef), BytesRef.deepCopyOf(spare));
termCount++;
}
@@ -249,7 +270,7 @@ public class MemoryPostingsFormat extend
fst = fst.pack(3, Math.max(10, fst.getNodeCount()/4));
}
fst.save(out);
- if (VERBOSE) System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer());
+ //System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer());
}
}
@@ -270,10 +291,7 @@ public class MemoryPostingsFormat extend
return new FieldsConsumer() {
@Override
public TermsConsumer addField(FieldInfo field) {
- if (field.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) {
- throw new UnsupportedOperationException("this codec cannot index offsets");
- }
- if (VERBOSE) System.out.println("\naddField field=" + field.name);
+ //System.out.println("\naddField field=" + field.name);
return new TermsWriter(out, field, doPackFST);
}
@@ -331,11 +349,9 @@ public class MemoryPostingsFormat extend
@Override
public int nextDoc() {
while(true) {
- if (VERBOSE) System.out.println(" nextDoc cycle docUpto=" + docUpto + " numDocs=" + numDocs + " fp=" + in.getPosition() + " this=" + this);
+ //System.out.println(" nextDoc cycle docUpto=" + docUpto + " numDocs=" + numDocs + " fp=" + in.getPosition() + " this=" + this);
if (docUpto == numDocs) {
- if (VERBOSE) {
- System.out.println(" END");
- }
+ // System.out.println(" END");
return docID = NO_MORE_DOCS;
}
docUpto++;
@@ -344,7 +360,7 @@ public class MemoryPostingsFormat extend
} else {
final int code = in.readVInt();
accum += code >>> 1;
- if (VERBOSE) System.out.println(" docID=" + accum + " code=" + code);
+ //System.out.println(" docID=" + accum + " code=" + code);
if ((code & 1) != 0) {
freq = 1;
} else {
@@ -352,8 +368,8 @@ public class MemoryPostingsFormat extend
assert freq > 0;
}
- if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
- // Skip positions
+ if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
+ // Skip positions/payloads
for(int posUpto=0;posUpto<freq;posUpto++) {
if (!storePayloads) {
in.readVInt();
@@ -365,11 +381,26 @@ public class MemoryPostingsFormat extend
in.skipBytes(payloadLen);
}
}
+ } else if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
+ // Skip positions/offsets/payloads
+ for(int posUpto=0;posUpto<freq;posUpto++) {
+ int posCode = in.readVInt();
+ if (storePayloads && ((posCode & 1) != 0)) {
+ payloadLen = in.readVInt();
+ }
+ if ((in.readVInt() & 1) != 0) {
+ // new offset length
+ in.readVInt();
+ }
+ if (storePayloads) {
+ in.skipBytes(payloadLen);
+ }
+ }
}
}
if (liveDocs == null || liveDocs.get(accum)) {
- if (VERBOSE) System.out.println(" return docID=" + accum + " freq=" + freq);
+ //System.out.println(" return docID=" + accum + " freq=" + freq);
return (docID = accum);
}
}
@@ -413,26 +444,30 @@ public class MemoryPostingsFormat extend
private int posPending;
private int payloadLength;
private boolean payloadRetrieved;
+ final boolean storeOffsets;
+ int offsetLength;
+ int startOffset;
private int pos;
private final BytesRef payload = new BytesRef();
- public FSTDocsAndPositionsEnum(boolean storePayloads) {
+ public FSTDocsAndPositionsEnum(boolean storePayloads, boolean storeOffsets) {
this.storePayloads = storePayloads;
+ this.storeOffsets = storeOffsets;
}
- public boolean canReuse(boolean storePayloads) {
- return storePayloads == this.storePayloads;
+ public boolean canReuse(boolean storePayloads, boolean storeOffsets) {
+ return storePayloads == this.storePayloads && storeOffsets == this.storeOffsets;
}
public FSTDocsAndPositionsEnum reset(BytesRef bufferIn, Bits liveDocs, int numDocs) {
assert numDocs > 0;
- if (VERBOSE) {
- System.out.println("D&P reset bytes this=" + this);
- for(int i=bufferIn.offset;i<bufferIn.length;i++) {
- System.out.println(" " + Integer.toHexString(bufferIn.bytes[i]&0xFF));
- }
- }
+
+ // System.out.println("D&P reset bytes this=" + this);
+ // for(int i=bufferIn.offset;i<bufferIn.length;i++) {
+ // System.out.println(" " + Integer.toHexString(bufferIn.bytes[i]&0xFF));
+ // }
+
if (buffer.length < bufferIn.length - bufferIn.offset) {
buffer = ArrayUtil.grow(buffer, bufferIn.length - bufferIn.offset);
}
@@ -447,6 +482,8 @@ public class MemoryPostingsFormat extend
this.numDocs = numDocs;
posPending = 0;
payloadRetrieved = false;
+ startOffset = storeOffsets ? 0 : -1; // always return -1 if no offsets are stored
+ offsetLength = 0;
return this;
}
@@ -456,9 +493,9 @@ public class MemoryPostingsFormat extend
nextPosition();
}
while(true) {
- if (VERBOSE) System.out.println(" nextDoc cycle docUpto=" + docUpto + " numDocs=" + numDocs + " fp=" + in.getPosition() + " this=" + this);
+ //System.out.println(" nextDoc cycle docUpto=" + docUpto + " numDocs=" + numDocs + " fp=" + in.getPosition() + " this=" + this);
if (docUpto == numDocs) {
- if (VERBOSE) System.out.println(" END");
+ //System.out.println(" END");
return docID = NO_MORE_DOCS;
}
docUpto++;
@@ -474,8 +511,9 @@ public class MemoryPostingsFormat extend
if (liveDocs == null || liveDocs.get(accum)) {
pos = 0;
+ startOffset = storeOffsets ? 0 : -1;
posPending = freq;
- if (VERBOSE) System.out.println(" return docID=" + accum + " freq=" + freq);
+ //System.out.println(" return docID=" + accum + " freq=" + freq);
return (docID = accum);
}
@@ -487,8 +525,18 @@ public class MemoryPostingsFormat extend
final int skipCode = in.readVInt();
if ((skipCode & 1) != 0) {
payloadLength = in.readVInt();
- if (VERBOSE) System.out.println(" new payloadLen=" + payloadLength);
+ //System.out.println(" new payloadLen=" + payloadLength);
}
+ }
+
+ if (storeOffsets) {
+ if ((in.readVInt() & 1) != 0) {
+ // new offset length
+ offsetLength = in.readVInt();
+ }
+ }
+
+ if (storePayloads) {
in.skipBytes(payloadLength);
}
}
@@ -497,7 +545,7 @@ public class MemoryPostingsFormat extend
@Override
public int nextPosition() {
- if (VERBOSE) System.out.println(" nextPos storePayloads=" + storePayloads + " this=" + this);
+ //System.out.println(" nextPos storePayloads=" + storePayloads + " this=" + this);
assert posPending > 0;
posPending--;
if (!storePayloads) {
@@ -511,6 +559,18 @@ public class MemoryPostingsFormat extend
//} else {
//System.out.println(" same payloadLen=" + payloadLength);
}
+ }
+
+ if (storeOffsets) {
+ int offsetCode = in.readVInt();
+ if ((offsetCode & 1) != 0) {
+ // new offset length
+ offsetLength = in.readVInt();
+ }
+ startOffset += offsetCode >>> 1;
+ }
+
+ if (storePayloads) {
payload.offset = in.getPosition();
in.skipBytes(payloadLength);
payload.length = payloadLength;
@@ -520,18 +580,18 @@ public class MemoryPostingsFormat extend
payloadRetrieved = false;
}
- if (VERBOSE) System.out.println(" pos=" + pos + " payload=" + payload + " fp=" + in.getPosition());
+ //System.out.println(" pos=" + pos + " payload=" + payload + " fp=" + in.getPosition());
return pos;
}
@Override
public int startOffset() {
- return -1;
+ return startOffset;
}
@Override
public int endOffset() {
- return -1;
+ return startOffset + offsetLength;
}
@Override
@@ -594,14 +654,14 @@ public class MemoryPostingsFormat extend
totalTermFreq = -1;
}
current.output.offset = buffer.getPosition();
- if (VERBOSE) System.out.println(" df=" + docFreq + " totTF=" + totalTermFreq + " offset=" + buffer.getPosition() + " len=" + current.output.length);
+ //System.out.println(" df=" + docFreq + " totTF=" + totalTermFreq + " offset=" + buffer.getPosition() + " len=" + current.output.length);
didDecode = true;
}
}
@Override
public boolean seekExact(BytesRef text, boolean useCache /* ignored */) throws IOException {
- if (VERBOSE) System.out.println("te.seekExact text=" + field.name + ":" + text.utf8ToString() + " this=" + this);
+ //System.out.println("te.seekExact text=" + field.name + ":" + text.utf8ToString() + " this=" + this);
current = fstEnum.seekExact(text);
didDecode = false;
return current != null;
@@ -609,25 +669,24 @@ public class MemoryPostingsFormat extend
@Override
public SeekStatus seekCeil(BytesRef text, boolean useCache /* ignored */) throws IOException {
- if (VERBOSE) System.out.println("te.seek text=" + field.name + ":" + text.utf8ToString() + " this=" + this);
+ //System.out.println("te.seek text=" + field.name + ":" + text.utf8ToString() + " this=" + this);
current = fstEnum.seekCeil(text);
if (current == null) {
return SeekStatus.END;
} else {
- if (VERBOSE) {
- System.out.println(" got term=" + current.input.utf8ToString());
- for(int i=0;i<current.output.length;i++) {
- System.out.println(" " + Integer.toHexString(current.output.bytes[i]&0xFF));
- }
- }
+
+ // System.out.println(" got term=" + current.input.utf8ToString());
+ // for(int i=0;i<current.output.length;i++) {
+ // System.out.println(" " + Integer.toHexString(current.output.bytes[i]&0xFF));
+ // }
didDecode = false;
if (text.equals(current.input)) {
- if (VERBOSE) System.out.println(" found!");
+ //System.out.println(" found!");
return SeekStatus.FOUND;
} else {
- if (VERBOSE) System.out.println(" not found: " + current.input.utf8ToString());
+ //System.out.println(" not found: " + current.input.utf8ToString());
return SeekStatus.NOT_FOUND;
}
}
@@ -654,9 +713,9 @@ public class MemoryPostingsFormat extend
@Override
public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
- if (needsOffsets) {
- // Not until we can index offsets...
- return null;
+ boolean hasOffsets = field.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ if (needsOffsets && !hasOffsets) {
+ return null; // not available
}
if (field.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
@@ -665,14 +724,14 @@ public class MemoryPostingsFormat extend
decodeMetaData();
FSTDocsAndPositionsEnum docsAndPositionsEnum;
if (reuse == null || !(reuse instanceof FSTDocsAndPositionsEnum)) {
- docsAndPositionsEnum = new FSTDocsAndPositionsEnum(field.storePayloads);
+ docsAndPositionsEnum = new FSTDocsAndPositionsEnum(field.storePayloads, hasOffsets);
} else {
docsAndPositionsEnum = (FSTDocsAndPositionsEnum) reuse;
- if (!docsAndPositionsEnum.canReuse(field.storePayloads)) {
- docsAndPositionsEnum = new FSTDocsAndPositionsEnum(field.storePayloads);
+ if (!docsAndPositionsEnum.canReuse(field.storePayloads, hasOffsets)) {
+ docsAndPositionsEnum = new FSTDocsAndPositionsEnum(field.storePayloads, hasOffsets);
}
}
- if (VERBOSE) System.out.println("D&P reset this=" + this);
+ //System.out.println("D&P reset this=" + this);
return docsAndPositionsEnum.reset(current.output, liveDocs, docFreq);
}
@@ -683,14 +742,14 @@ public class MemoryPostingsFormat extend
@Override
public BytesRef next() throws IOException {
- if (VERBOSE) System.out.println("te.next");
+ //System.out.println("te.next");
current = fstEnum.next();
if (current == null) {
- if (VERBOSE) System.out.println(" END");
+ //System.out.println(" END");
return null;
}
didDecode = false;
- if (VERBOSE) System.out.println(" term=" + field.name + ":" + current.input.utf8ToString());
+ //System.out.println(" term=" + field.name + ":" + current.input.utf8ToString());
return current.input;
}
@@ -794,9 +853,7 @@ public class MemoryPostingsFormat extend
break;
}
final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount);
- if (VERBOSE) {
- System.out.println("load field=" + termsReader.field.name);
- }
+ // System.out.println("load field=" + termsReader.field.name);
fields.put(termsReader.field.name, termsReader);
}
} finally {
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java Tue Mar 13 16:43:38 2012
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.Map;
+import java.util.ServiceLoader; // javadocs
import java.util.Set;
import java.util.TreeMap;
@@ -47,7 +48,14 @@ import org.apache.lucene.util.IOUtils;
/**
* Enables per field format support.
- *
+ * <p>
+ * Note, when extending this class, the name ({@link #getName}) is
+ * written into the index. In order for the field to be read, the
+ * name must resolve to your implementation via {@link #forName(String)}.
+ * This method uses Java's
+ * {@link ServiceLoader Service Provider Interface} to resolve format names.
+ * <p>
+ * @see ServiceLoader
* @lucene.experimental
*/
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/sep/SepDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/sep/SepDocValuesConsumer.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/sep/SepDocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/sep/SepDocValuesConsumer.java Tue Mar 13 16:43:38 2012
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
+import org.apache.lucene.codecs.PerDocProducerBase;
import org.apache.lucene.codecs.lucene40.values.DocValuesWriterBase;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
@@ -58,7 +59,7 @@ public class SepDocValuesConsumer extend
private static void files(Directory dir,FieldInfos fieldInfos, String segmentName, Set<String> files) {
for (FieldInfo fieldInfo : fieldInfos) {
if (fieldInfo.hasDocValues()) {
- String filename = docValuesId(segmentName, fieldInfo.number);
+ String filename = PerDocProducerBase.docValuesId(segmentName, fieldInfo.number);
switch (fieldInfo.getDocValuesType()) {
case BYTES_FIXED_DEREF:
case BYTES_VAR_DEREF:
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/sep/SepDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/sep/SepDocValuesProducer.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/sep/SepDocValuesProducer.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/sep/SepDocValuesProducer.java Tue Mar 13 16:43:38 2012
@@ -22,16 +22,22 @@ import java.util.Collection;
import java.util.Map;
import java.util.TreeMap;
-import org.apache.lucene.codecs.lucene40.values.DocValuesReaderBase;
+import org.apache.lucene.codecs.PerDocProducerBase;
+import org.apache.lucene.codecs.lucene40.values.Bytes;
+import org.apache.lucene.codecs.lucene40.values.Floats;
+import org.apache.lucene.codecs.lucene40.values.Ints;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.IOUtils;
/**
* Implementation of PerDocProducer that uses separate files.
* @lucene.experimental
*/
-public class SepDocValuesProducer extends DocValuesReaderBase {
+public class SepDocValuesProducer extends PerDocProducerBase {
private final TreeMap<String, DocValues> docValues;
/**
@@ -51,4 +57,35 @@ public class SepDocValuesProducer extend
protected void closeInternal(Collection<? extends Closeable> closeables) throws IOException {
IOUtils.close(closeables);
}
+
+ @Override
+ protected DocValues loadDocValues(int docCount, Directory dir, String id,
+ Type type, IOContext context) throws IOException {
+ switch (type) {
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
+ case VAR_INTS:
+ return Ints.getValues(dir, id, docCount, type, context);
+ case FLOAT_32:
+ return Floats.getValues(dir, id, docCount, context, type);
+ case FLOAT_64:
+ return Floats.getValues(dir, id, docCount, context, type);
+ case BYTES_FIXED_STRAIGHT:
+ return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, getComparator(), context);
+ case BYTES_FIXED_DEREF:
+ return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, getComparator(), context);
+ case BYTES_FIXED_SORTED:
+ return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount, getComparator(), context);
+ case BYTES_VAR_STRAIGHT:
+ return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, getComparator(), context);
+ case BYTES_VAR_DEREF:
+ return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, getComparator(), context);
+ case BYTES_VAR_SORTED:
+ return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount, getComparator(), context);
+ default:
+ throw new IllegalStateException("unrecognized index values mode " + type);
+ }
+ }
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCodec.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCodec.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCodec.java Tue Mar 13 16:43:38 2012
@@ -26,7 +26,6 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.codecs.SegmentInfosFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40DocValuesFormat;
/**
* plain text index format.
@@ -41,7 +40,7 @@ public final class SimpleTextCodec exten
private final FieldInfosFormat fieldInfosFormat = new SimpleTextFieldInfosFormat();
private final TermVectorsFormat vectorsFormat = new SimpleTextTermVectorsFormat();
// TODO: need a plain-text impl
- private final DocValuesFormat docValues = new Lucene40DocValuesFormat();
+ private final DocValuesFormat docValues = new SimpleTextDocValuesFormat();
// TODO: need a plain-text impl (using the above)
private final NormsFormat normsFormat = new SimpleTextNormsFormat();
private final LiveDocsFormat liveDocs = new SimpleTextLiveDocsFormat();
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java Tue Mar 13 16:43:38 2012
@@ -18,35 +18,123 @@ package org.apache.lucene.codecs.simplet
*/
import java.io.IOException;
+import java.util.Comparator;
+import java.util.HashSet;
import java.util.Set;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PerDocConsumer;
import org.apache.lucene.codecs.PerDocProducer;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValues.Type;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
/**
* plain-text norms format
* <p>
* <b><font color="red">FOR RECREATIONAL USE ONLY</font></B>
+ *
* @lucene.experimental
*/
public class SimpleTextNormsFormat extends NormsFormat {
+ private static final String NORMS_SEG_SUFFIX = "len";
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new SimpleTextNormsConsumer(state.directory, state.segmentName, state.context);
+ return new SimpleTextNormsPerDocConsumer(state, NORMS_SEG_SUFFIX);
}
-
+
@Override
public PerDocProducer docsProducer(SegmentReadState state) throws IOException {
- return new SimpleTextNormsProducer(state.dir, state.segmentInfo, state.fieldInfos, state.context);
+ return new SimpleTextNormsPerDocProducer(state,
+ BytesRef.getUTF8SortedAsUnicodeComparator(), NORMS_SEG_SUFFIX);
}
-
+
@Override
public void files(SegmentInfo info, Set<String> files) throws IOException {
- SimpleTextNormsConsumer.files(info, files);
- }
+ SimpleTextNormsPerDocConsumer.files(info, files);
+ }
+
+ public static class SimpleTextNormsPerDocProducer extends
+ SimpleTextPerDocProducer {
+
+ public SimpleTextNormsPerDocProducer(SegmentReadState state,
+ Comparator<BytesRef> comp, String segmentSuffix) throws IOException {
+ super(state, comp, segmentSuffix);
+ }
+
+ @Override
+ protected boolean canLoad(FieldInfo info) {
+ return info.hasNorms();
+ }
+
+ @Override
+ protected Type getDocValuesType(FieldInfo info) {
+ return info.getNormType();
+ }
+
+ @Override
+ protected boolean anyDocValuesFields(FieldInfos infos) {
+ return infos.hasNorms();
+ }
+
+ }
+
+ public static class SimpleTextNormsPerDocConsumer extends
+ SimpleTextPerDocConsumer {
+
+ public SimpleTextNormsPerDocConsumer(PerDocWriteState state,
+ String segmentSuffix) throws IOException {
+ super(state, segmentSuffix);
+ }
+
+ @Override
+ protected DocValues getDocValuesForMerge(AtomicReader reader, FieldInfo info)
+ throws IOException {
+ return reader.normValues(info.name);
+ }
+
+ @Override
+ protected boolean canMerge(FieldInfo info) {
+ return info.hasNorms();
+ }
+
+ @Override
+ protected Type getDocValuesType(FieldInfo info) {
+ return info.getNormType();
+ }
+
+ @Override
+ public void abort() {
+ Set<String> files = new HashSet<String>();
+ filesInternal(state.fieldInfos, state.segmentName, files, segmentSuffix);
+ IOUtils.deleteFilesIgnoringExceptions(state.directory,
+ files.toArray(new String[0]));
+ }
+
+ public static void files(SegmentInfo segmentInfo, Set<String> files)
+ throws IOException {
+ filesInternal(segmentInfo.getFieldInfos(), segmentInfo.name, files,
+ NORMS_SEG_SUFFIX);
+ }
+
+ public static void filesInternal(FieldInfos fieldInfos, String segmentName,
+ Set<String> files, String segmentSuffix) {
+ for (FieldInfo fieldInfo : fieldInfos) {
+ if (fieldInfo.hasNorms()) {
+ String id = docValuesId(segmentName, fieldInfo.number);
+ files.add(IndexFileNames.segmentFileName(id, "",
+ segmentSuffix));
+ }
+ }
+ }
+ }
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java Tue Mar 13 16:43:38 2012
@@ -60,12 +60,17 @@ public abstract class AtomicReader exten
return readerContext;
}
- /** Returns true if there are norms stored for this field. */
- public boolean hasNorms(String field) throws IOException {
- // backward compatible implementation.
- // SegmentReader has an efficient implementation.
+ /**
+ * Returns true if there are norms stored for this field.
+ * @deprecated (4.0) use {@link #getFieldInfos()} and check {@link FieldInfo#hasNorms()}
+ * for the field instead.
+ */
+ @Deprecated
+ public final boolean hasNorms(String field) throws IOException {
ensureOpen();
- return normValues(field) != null;
+ // note: using normValues(field) != null would potentially cause i/o
+ FieldInfo fi = getFieldInfos().fieldInfo(field);
+ return fi != null && fi.hasNorms();
}
/**
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/BaseCompositeReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/BaseCompositeReader.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/BaseCompositeReader.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/BaseCompositeReader.java Tue Mar 13 16:43:38 2012
@@ -53,6 +53,14 @@ public abstract class BaseCompositeReade
private final int numDocs;
private final boolean hasDeletions;
+ /**
+ * Constructs a {@code BaseCompositeReader} on the given subReaders.
+ * @param subReaders the wrapped sub-readers. This array is returned by
+ * {@link #getSequentialSubReaders} and used to resolve the correct
+ * subreader for docID-based methods. <b>Please note:</b> This array is <b>not</b>
+ * cloned and not protected for modification, the subclass is responsible
+ * to do this.
+ */
protected BaseCompositeReader(R[] subReaders) throws IOException {
this.subReaders = subReaders;
starts = new int[subReaders.length + 1]; // build starts array
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java Tue Mar 13 16:43:38 2012
@@ -210,7 +210,7 @@ class BufferedDeletesStream {
// Lock order: IW -> BD -> RP
assert readerPool.infoIsLive(info);
- final IndexWriter.ReadersAndLiveDocs rld = readerPool.get(info, true);
+ final ReadersAndLiveDocs rld = readerPool.get(info, true);
final SegmentReader reader = rld.getReader(IOContext.READ);
int delCount = 0;
final boolean segAllDeletes;
@@ -224,11 +224,12 @@ class BufferedDeletesStream {
// Don't delete by Term here; DocumentsWriterPerThread
// already did that on flush:
delCount += applyQueryDeletes(packet.queriesIterable(), rld, reader);
- final int fullDelCount = rld.info.getDelCount() + rld.pendingDeleteCount;
+ final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount();
assert fullDelCount <= rld.info.docCount;
segAllDeletes = fullDelCount == rld.info.docCount;
} finally {
- readerPool.release(reader, false);
+ rld.release(reader);
+ readerPool.release(rld);
}
anyNewDeletes |= delCount > 0;
@@ -262,18 +263,19 @@ class BufferedDeletesStream {
if (coalescedDeletes != null) {
// Lock order: IW -> BD -> RP
assert readerPool.infoIsLive(info);
- final IndexWriter.ReadersAndLiveDocs rld = readerPool.get(info, true);
+ final ReadersAndLiveDocs rld = readerPool.get(info, true);
final SegmentReader reader = rld.getReader(IOContext.READ);
int delCount = 0;
final boolean segAllDeletes;
try {
delCount += applyTermDeletes(coalescedDeletes.termsIterable(), rld, reader);
delCount += applyQueryDeletes(coalescedDeletes.queriesIterable(), rld, reader);
- final int fullDelCount = rld.info.getDelCount() + rld.pendingDeleteCount;
+ final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount();
assert fullDelCount <= rld.info.docCount;
segAllDeletes = fullDelCount == rld.info.docCount;
- } finally {
- readerPool.release(reader, false);
+ } finally {
+ rld.release(reader);
+ readerPool.release(rld);
}
anyNewDeletes |= delCount > 0;
@@ -353,7 +355,7 @@ class BufferedDeletesStream {
}
// Delete by Term
- private synchronized long applyTermDeletes(Iterable<Term> termsIter, IndexWriter.ReadersAndLiveDocs rld, SegmentReader reader) throws IOException {
+ private synchronized long applyTermDeletes(Iterable<Term> termsIter, ReadersAndLiveDocs rld, SegmentReader reader) throws IOException {
long delCount = 0;
Fields fields = reader.fields();
if (fields == null) {
@@ -394,7 +396,7 @@ class BufferedDeletesStream {
// System.out.println(" term=" + term);
if (termsEnum.seekExact(term.bytes(), false)) {
- DocsEnum docsEnum = termsEnum.docs(rld.liveDocs, docs, false);
+ DocsEnum docsEnum = termsEnum.docs(rld.getLiveDocs(), docs, false);
//System.out.println("BDS: got docsEnum=" + docsEnum);
if (docsEnum != null) {
@@ -434,7 +436,7 @@ class BufferedDeletesStream {
}
// Delete by query
- private static long applyQueryDeletes(Iterable<QueryAndLimit> queriesIter, IndexWriter.ReadersAndLiveDocs rld, final SegmentReader reader) throws IOException {
+ private static long applyQueryDeletes(Iterable<QueryAndLimit> queriesIter, ReadersAndLiveDocs rld, final SegmentReader reader) throws IOException {
long delCount = 0;
final AtomicReaderContext readerContext = reader.getTopReaderContext();
boolean any = false;
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Tue Mar 13 16:43:38 2012
@@ -651,28 +651,17 @@ public class CheckIndex {
if (infoStream != null) {
infoStream.print(" test: field norms.........");
}
- DocValues dv;
for (FieldInfo info : fieldInfos) {
- if (reader.hasNorms(info.name)) {
- dv = reader.normValues(info.name);
- assert dv != null;
- if (dv.getSource().hasArray()) {
- Object array = dv.getSource().getArray();
- if (Array.getLength(array) != reader.maxDoc()) {
- throw new RuntimeException("norms for field: " + info.name + " are of the wrong size");
- }
- }
- if (!info.isIndexed || info.omitNorms) {
- throw new RuntimeException("field: " + info.name + " should omit norms but has them!");
- }
+ if (info.hasNorms()) {
+ assert reader.hasNorms(info.name); // deprecated path
+ DocValues dv = reader.normValues(info.name);
+ checkDocValues(dv, info.name, info.getNormType(), reader.maxDoc());
++status.totFields;
} else {
+ assert !reader.hasNorms(info.name); // deprecated path
if (reader.normValues(info.name) != null) {
throw new RuntimeException("field: " + info.name + " should omit norms but has them!");
}
- if (info.normsPresent()) {
- throw new RuntimeException("field: " + info.name + " should have norms but omits them!");
- }
}
}
@@ -1171,6 +1160,92 @@ public class CheckIndex {
return status;
}
+ /** Helper method to verify values (either docvalues or norms), also checking
+ * type and size against fieldinfos/segmentinfo
+ */
+ private void checkDocValues(DocValues docValues, String fieldName, DocValues.Type expectedType, int expectedDocs) throws IOException {
+ if (docValues == null) {
+ throw new RuntimeException("field: " + fieldName + " omits docvalues but should have them!");
+ }
+ DocValues.Type type = docValues.getType();
+ if (type != expectedType) {
+ throw new RuntimeException("field: " + fieldName + " has type: " + type + " but fieldInfos says:" + expectedType);
+ }
+ final Source values = docValues.getDirectSource();
+ int size = docValues.getValueSize();
+ for (int i = 0; i < expectedDocs; i++) {
+ switch (type) {
+ case BYTES_FIXED_SORTED:
+ case BYTES_VAR_SORTED:
+ case BYTES_FIXED_DEREF:
+ case BYTES_FIXED_STRAIGHT:
+ case BYTES_VAR_DEREF:
+ case BYTES_VAR_STRAIGHT:
+ BytesRef bytes = new BytesRef();
+ values.getBytes(i, bytes);
+ if (size != -1 && size != bytes.length) {
+ throw new RuntimeException("field: " + fieldName + " returned wrongly sized bytes, was: " + bytes.length + " should be: " + size);
+ }
+ break;
+ case FLOAT_32:
+ assert size == 4;
+ values.getFloat(i);
+ break;
+ case FLOAT_64:
+ assert size == 8;
+ values.getFloat(i);
+ break;
+ case VAR_INTS:
+ assert size == -1;
+ values.getInt(i);
+ break;
+ case FIXED_INTS_16:
+ assert size == 2;
+ values.getInt(i);
+ break;
+ case FIXED_INTS_32:
+ assert size == 4;
+ values.getInt(i);
+ break;
+ case FIXED_INTS_64:
+ assert size == 8;
+ values.getInt(i);
+ break;
+ case FIXED_INTS_8:
+ assert size == 1;
+ values.getInt(i);
+ break;
+ default:
+ throw new IllegalArgumentException("Field: " + fieldName
+ + " - no such DocValues type: " + type);
+ }
+ }
+ if (type == DocValues.Type.BYTES_FIXED_SORTED || type == DocValues.Type.BYTES_VAR_SORTED) {
+ // check sorted bytes
+ SortedSource sortedValues = values.asSortedSource();
+ Comparator<BytesRef> comparator = sortedValues.getComparator();
+ int lastOrd = -1;
+ BytesRef lastBytes = new BytesRef();
+ for (int i = 0; i < expectedDocs; i++) {
+ int ord = sortedValues.ord(i);
+ if (ord < 0 || ord > expectedDocs) {
+ throw new RuntimeException("field: " + fieldName + " ord is out of bounds: " + ord);
+ }
+ BytesRef bytes = new BytesRef();
+ sortedValues.getByOrd(ord, bytes);
+ if (lastOrd != -1) {
+ int ordComp = Integer.signum(new Integer(ord).compareTo(new Integer(lastOrd)));
+ int bytesComp = Integer.signum(comparator.compare(bytes, lastBytes));
+ if (ordComp != bytesComp) {
+ throw new RuntimeException("field: " + fieldName + " ord comparison is wrong: " + ordComp + " comparator claims: " + bytesComp);
+ }
+ }
+ lastOrd = ord;
+ lastBytes = bytes;
+ }
+ }
+ }
+
private Status.DocValuesStatus testDocValues(SegmentInfo info,
SegmentReader reader) {
final Status.DocValuesStatus status = new Status.DocValuesStatus();
@@ -1183,87 +1258,7 @@ public class CheckIndex {
if (fieldInfo.hasDocValues()) {
status.totalValueFields++;
final DocValues docValues = reader.docValues(fieldInfo.name);
- if (docValues == null) {
- throw new RuntimeException("field: " + fieldInfo.name + " omits docvalues but should have them!");
- }
- DocValues.Type type = docValues.type();
- if (type != fieldInfo.getDocValuesType()) {
- throw new RuntimeException("field: " + fieldInfo.name + " has type: " + type + " but fieldInfos says:" + fieldInfo.getDocValuesType());
- }
- final Source values = docValues.getDirectSource();
- final int maxDoc = reader.maxDoc();
- int size = docValues.getValueSize();
- for (int i = 0; i < maxDoc; i++) {
- switch (fieldInfo.getDocValuesType()) {
- case BYTES_FIXED_SORTED:
- case BYTES_VAR_SORTED:
- case BYTES_FIXED_DEREF:
- case BYTES_FIXED_STRAIGHT:
- case BYTES_VAR_DEREF:
- case BYTES_VAR_STRAIGHT:
- BytesRef bytes = new BytesRef();
- values.getBytes(i, bytes);
- if (size != -1 && size != bytes.length) {
- throw new RuntimeException("field: " + fieldInfo.name + " returned wrongly sized bytes, was: " + bytes.length + " should be: " + size);
- }
- break;
- case FLOAT_32:
- assert size == 4;
- values.getFloat(i);
- break;
- case FLOAT_64:
- assert size == 8;
- values.getFloat(i);
- break;
- case VAR_INTS:
- assert size == -1;
- values.getInt(i);
- break;
- case FIXED_INTS_16:
- assert size == 2;
- values.getInt(i);
- break;
- case FIXED_INTS_32:
- assert size == 4;
- values.getInt(i);
- break;
- case FIXED_INTS_64:
- assert size == 8;
- values.getInt(i);
- break;
- case FIXED_INTS_8:
- assert size == 1;
- values.getInt(i);
- break;
- default:
- throw new IllegalArgumentException("Field: " + fieldInfo.name
- + " - no such DocValues type: " + fieldInfo.getDocValuesType());
- }
- }
- if (type == DocValues.Type.BYTES_FIXED_SORTED || type == DocValues.Type.BYTES_VAR_SORTED) {
- // check sorted bytes
- SortedSource sortedValues = values.asSortedSource();
- Comparator<BytesRef> comparator = sortedValues.getComparator();
- int lastOrd = -1;
- BytesRef lastBytes = new BytesRef();
- for (int i = 0; i < maxDoc; i++) {
- int ord = sortedValues.ord(i);
- if (ord < 0 || ord > maxDoc) {
- throw new RuntimeException("field: " + fieldInfo.name + " ord is out of bounds: " + ord);
- }
- BytesRef bytes = new BytesRef();
- sortedValues.getByOrd(ord, bytes);
- if (lastOrd != -1) {
- int ordComp = Integer.signum(new Integer(ord).compareTo(new Integer(lastOrd)));
- int bytesComp = Integer.signum(comparator.compare(bytes, lastBytes));
- if (ordComp != bytesComp) {
- throw new RuntimeException("field: " + fieldInfo.name + " ord comparison is wrong: " + ordComp + " comparator claims: " + bytesComp);
- }
- }
- lastOrd = ord;
- lastBytes = bytes;
- }
- }
+ checkDocValues(docValues, fieldInfo.name, fieldInfo.getDocValuesType(), reader.maxDoc());
} else {
if (reader.docValues(fieldInfo.name) != null) {
throw new RuntimeException("field: " + fieldInfo.name + " has docvalues but should omit them!");
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java Tue Mar 13 16:43:38 2012
@@ -81,6 +81,9 @@ public abstract class CompositeReader ex
* If this method returns an empty array, that means this
* reader is a null reader (for example a MultiReader
* that has no sub readers).
+ * <p><b>Warning:</b> Don't modify the returned array!
+ * Doing so will corrupt the internal structure of this
+ * {@code CompositeReader}.
*/
public abstract IndexReader[] getSequentialSubReaders();
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java Tue Mar 13 16:43:38 2012
@@ -323,8 +323,17 @@ public abstract class DirectoryReader ex
}
}
- protected DirectoryReader(Directory directory, AtomicReader[] readers) throws CorruptIndexException, IOException {
- super(readers);
+ /**
+ * Expert: Constructs a {@code DirectoryReader} on the given subReaders.
+ * @param segmentReaders the wrapped atomic index segment readers. This array is
+ * returned by {@link #getSequentialSubReaders} and used to resolve the correct
+ * subreader for docID-based methods. <b>Please note:</b> This array is <b>not</b>
+ * cloned and not protected for modification outside of this reader.
+ * Subclasses of {@code DirectoryReader} should take care to not allow
+ * modification of this internal array, e.g. {@link #doOpenIfChanged()}.
+ */
+ protected DirectoryReader(Directory directory, AtomicReader[] segmentReaders) throws CorruptIndexException, IOException {
+ super(segmentReaders);
this.directory = directory;
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java Tue Mar 13 16:43:38 2012
@@ -216,6 +216,13 @@ public class DocTermOrds {
}
}
+ /**
+ * @return The number of terms in this field
+ */
+ public int numTerms() {
+ return numTermsInField;
+ }
+
/** Subclass can override this */
protected void visitTerm(TermsEnum te, int termNum) throws IOException {
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/DocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/DocValues.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/DocValues.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/DocValues.java Tue Mar 13 16:43:38 2012
@@ -90,7 +90,7 @@ public abstract class DocValues implemen
/**
* Returns the {@link Type} of this {@link DocValues} instance
*/
- public abstract Type type();
+ public abstract Type getType();
/**
* Closes this {@link DocValues} instance. This method should only be called
@@ -191,7 +191,7 @@ public abstract class DocValues implemen
*
* @return the {@link Type} of this source.
*/
- public Type type() {
+ public Type getType() {
return type;
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Tue Mar 13 16:43:38 2012
@@ -122,14 +122,23 @@ public final class FieldInfo {
}
}
+ /**
+ * @return true if this field has any docValues.
+ */
public boolean hasDocValues() {
return docValueType != null;
}
+ /**
+ * @return {@link DocValues.Type} of the docValues. this may be null if the field has no docvalues.
+ */
public DocValues.Type getDocValuesType() {
return docValueType;
}
+ /**
+ * @return {@link DocValues.Type} of the norm. this may be null if the field has no norms.
+ */
public DocValues.Type getNormType() {
return normType;
}
@@ -146,11 +155,17 @@ public final class FieldInfo {
}
}
+ /**
+ * @return true if norms are explicitly omitted for this field
+ */
public boolean omitNorms() {
return omitNorms;
}
- public boolean normsPresent() {
+ /**
+ * @return true if this field actually has any norms.
+ */
+ public boolean hasNorms() {
return isIndexed && !omitNorms && normType != null;
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Tue Mar 13 16:43:38 2012
@@ -178,7 +178,7 @@ public final class FieldInfos implements
return fis;
}
- /** Returns true if any fields do not positions */
+ /** Returns true if any fields have positions */
public boolean hasProx() {
if (isReadOnly()) {
return hasProx;
@@ -349,6 +349,12 @@ public final class FieldInfos implements
return fi;
}
+ /**
+ * lookup the number of a field by name.
+ *
+ * @param fieldName field's name
+ * @return number of field, or -1 if it does not exist.
+ */
public int fieldNumber(String fieldName) {
FieldInfo fi = fieldInfo(fieldName);
return (fi != null) ? fi.number : -1;
@@ -384,11 +390,17 @@ public final class FieldInfos implements
return byNumber.values().iterator();
}
+ /**
+ * @return number of fields
+ */
public int size() {
assert byNumber.size() == byName.size();
return byNumber.size();
}
+ /**
+ * @return true if at least one field has any vectors
+ */
public boolean hasVectors() {
if (isReadOnly()) {
return hasVectors;
@@ -402,9 +414,12 @@ public final class FieldInfos implements
return false;
}
+ /**
+ * @return true if at least one field has any norms
+ */
public boolean hasNorms() {
for (FieldInfo fi : this) {
- if (fi.normsPresent()) {
+ if (fi.hasNorms()) {
return true;
}
}
@@ -441,7 +456,10 @@ public final class FieldInfos implements
return roFis;
}
- public boolean anyDocValuesFields() {
+ /**
+ * @return true if at least one field has docValues
+ */
+ public boolean hasDocValues() {
for (FieldInfo fi : this) {
if (fi.hasDocValues()) {
return true;
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java Tue Mar 13 16:43:38 2012
@@ -360,12 +360,6 @@ public class FilterAtomicReader extends
}
@Override
- public boolean hasNorms(String field) throws IOException {
- ensureOpen();
- return in.hasNorms(field);
- }
-
- @Override
protected void doClose() throws IOException {
in.close();
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java Tue Mar 13 16:43:38 2012
@@ -453,7 +453,7 @@ final class IndexFileDeleter {
assert Thread.holdsLock(writer);
if (infoStream.isEnabled("IFD")) {
- infoStream.message("IFD", "now checkpoint \"" + writer.segString(segmentInfos) + "\" [" + segmentInfos.size() + " segments " + "; isCommit = " + isCommit + "]");
+ infoStream.message("IFD", "now checkpoint \"" + writer.segString(writer.toLiveInfos(segmentInfos)) + "\" [" + segmentInfos.size() + " segments " + "; isCommit = " + isCommit + "]");
}
// Try again now to delete any previously un-deletable
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java?rev=1300232&r1=1300231&r2=1300232&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java Tue Mar 13 16:43:38 2012
@@ -41,25 +41,27 @@ public abstract class IndexReaderContext
this.isTopLevel = parent==null;
}
+ /** Returns the {@link IndexReader}, this context represents. */
public abstract IndexReader reader();
/**
* Returns the context's leaves if this context is a top-level context
* otherwise <code>null</code>. For convenience, if this is an
* {@link AtomicReaderContext} this returns itsself as the only leaf.
- * <p>
- * Note: this is convenience method since leaves can always be obtained by
+ * <p>Note: this is convenience method since leaves can always be obtained by
* walking the context tree.
+ * <p><b>Warning:</b> Don't modify the returned array!
+ * Doing so will corrupt the internal structure of this
+ * {@code IndexReaderContext}.
*/
public abstract AtomicReaderContext[] leaves();
/**
* Returns the context's children iff this context is a composite context
* otherwise <code>null</code>.
- * <p>
- * Note: this method is a convenience method to prevent
- * <code>instanceof</code> checks and type-casts to
- * {@link CompositeReaderContext}.
+ * <p><b>Warning:</b> Don't modify the returned array!
+ * Doing so will corrupt the internal structure of this
+ * {@code IndexReaderContext}.
*/
public abstract IndexReaderContext[] children();
}
\ No newline at end of file