You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2011/09/29 23:49:50 UTC
svn commit: r1177441 [2/6] - in /lucene/dev/branches/solrcloud: ./
dev-tools/idea/lucene/contrib/ lucene/
lucene/contrib/demo/src/java/org/apache/lucene/demo/
lucene/contrib/demo/src/java/org/apache/lucene/demo/xmlparser/
lucene/contrib/highlighter/src...
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java Thu Sep 29 21:49:45 2011
@@ -86,7 +86,7 @@ public abstract class Tokenizer extends
}
/** Expert: Reset the tokenizer to a new reader. Typically, an
- * analyzer (in its reusableTokenStream method) will use
+ * analyzer (in its tokenStream method) will use
* this to re-use a previously created tokenizer. */
public void reset(Reader input) throws IOException {
this.input = input;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/BinaryField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/BinaryField.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/BinaryField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/BinaryField.java Thu Sep 29 21:49:45 2011
@@ -31,16 +31,16 @@ public final class BinaryField extends F
/** Creates a new BinaryField */
public BinaryField(String name, byte[] value) {
- super(name, BinaryField.TYPE_STORED, value);
+ super(name, value, BinaryField.TYPE_STORED);
}
/** Creates a new BinaryField */
public BinaryField(String name, byte[] value, int offset, int length) {
- super(name, BinaryField.TYPE_STORED, value, offset, length);
+ super(name, value, offset, length, BinaryField.TYPE_STORED);
}
/** Creates a new BinaryField */
public BinaryField(String name, BytesRef bytes) {
- super(name, BinaryField.TYPE_STORED, bytes);
+ super(name, bytes, BinaryField.TYPE_STORED);
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/Document.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/Document.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/Document.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/Document.java Thu Sep 29 21:49:45 2011
@@ -209,6 +209,32 @@ public final class Document implements I
return fields;
}
+ private final static String[] NO_STRINGS = new String[0];
+
+ /**
+ * Returns an array of values of the field specified as the method parameter.
+ * This method returns an empty array when there are no
+ * matching fields. It never returns null.
+ * For {@link NumericField}s it returns the string value of the number. If you want
+ * the actual {@code NumericField} instances back, use {@link #getFields}.
+ * @param name the name of the field
+ * @return a <code>String[]</code> of field values
+ */
+ public final String[] getValues(String name) {
+ List<String> result = new ArrayList<String>();
+ for (IndexableField field : fields) {
+ if (field.name().equals(name) && field.stringValue() != null) {
+ result.add(field.stringValue());
+ }
+ }
+
+ if (result.size() == 0) {
+ return NO_STRINGS;
+ }
+
+ return result.toArray(new String[result.size()]);
+ }
+
/** Returns the string value of the field with the given name if any exist in
* this document, or null. If multiple fields exist with this name, this
* method returns the first value added. If only binary fields with this name
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/Field.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/Field.java Thu Sep 29 21:49:45 2011
@@ -60,7 +60,7 @@ public class Field implements IndexableF
this.type = type;
}
- public Field(String name, IndexableFieldType type, Reader reader) {
+ public Field(String name, Reader reader, IndexableFieldType type) {
if (name == null) {
throw new NullPointerException("name cannot be null");
}
@@ -76,7 +76,7 @@ public class Field implements IndexableF
this.type = type;
}
- public Field(String name, IndexableFieldType type, TokenStream tokenStream) {
+ public Field(String name, TokenStream tokenStream, IndexableFieldType type) {
if (name == null) {
throw new NullPointerException("name cannot be null");
}
@@ -93,15 +93,15 @@ public class Field implements IndexableF
this.type = type;
}
- public Field(String name, IndexableFieldType type, byte[] value) {
- this(name, type, value, 0, value.length);
+ public Field(String name, byte[] value, IndexableFieldType type) {
+ this(name, value, 0, value.length, type);
}
- public Field(String name, IndexableFieldType type, byte[] value, int offset, int length) {
- this(name, type, new BytesRef(value, offset, length));
+ public Field(String name, byte[] value, int offset, int length, IndexableFieldType type) {
+ this(name, new BytesRef(value, offset, length), type);
}
- public Field(String name, IndexableFieldType type, BytesRef bytes) {
+ public Field(String name, BytesRef bytes, IndexableFieldType type) {
if (type.indexed() && !type.tokenized()) {
throw new IllegalArgumentException("Non-tokenized fields must use String values");
}
@@ -111,7 +111,7 @@ public class Field implements IndexableF
this.name = name;
}
- public Field(String name, IndexableFieldType type, String value) {
+ public Field(String name, String value, IndexableFieldType type) {
if (name == null) {
throw new IllegalArgumentException("name cannot be null");
}
@@ -350,9 +350,9 @@ public class Field implements IndexableF
if (tokenStream != null) {
return tokenStream;
} else if (readerValue() != null) {
- return analyzer.reusableTokenStream(name(), readerValue());
+ return analyzer.tokenStream(name(), readerValue());
} else if (stringValue() != null) {
- return analyzer.reusableTokenStream(name(), new StringReader(stringValue()));
+ return analyzer.tokenStream(name(), new StringReader(stringValue()));
}
throw new IllegalArgumentException("Field must have either TokenStream, String or Reader value");
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/StringField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/StringField.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/StringField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/StringField.java Thu Sep 29 21:49:45 2011
@@ -54,7 +54,7 @@ public final class StringField extends F
/** Creates a new un-stored StringField */
public StringField(String name, String value) {
- super(name, TYPE_UNSTORED, value);
+ super(name, value, TYPE_UNSTORED);
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/TextField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/TextField.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/TextField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/TextField.java Thu Sep 29 21:49:45 2011
@@ -50,16 +50,16 @@ public final class TextField extends Fie
/** Creates a new un-stored TextField */
public TextField(String name, Reader reader) {
- super(name, TextField.TYPE_UNSTORED, reader);
+ super(name, reader, TextField.TYPE_UNSTORED);
}
/** Creates a new un-stored TextField */
public TextField(String name, String value) {
- super(name, TextField.TYPE_UNSTORED, value);
+ super(name, value, TextField.TYPE_UNSTORED);
}
/** Creates a new un-stored TextField */
public TextField(String name, TokenStream stream) {
- super(name, TextField.TYPE_UNSTORED, stream);
+ super(name, stream, TextField.TYPE_UNSTORED);
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Thu Sep 29 21:49:45 2011
@@ -83,9 +83,9 @@ final class DocFieldProcessor extends Do
// FieldInfo.storePayload.
final String fileName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.FIELD_INFOS_EXTENSION);
state.fieldInfos.write(state.directory, fileName);
- for (DocValuesConsumer consumers : docValues.values()) {
- consumers.finish(state.numDocs);
- };
+ for (DocValuesConsumerAndDocID consumers : docValues.values()) {
+ consumers.docValuesConsumer.finish(state.numDocs);
+ }
// close perDocConsumer during flush to ensure all files are flushed due to PerCodec CFS
IOUtils.close(perDocConsumers.values());
}
@@ -297,14 +297,28 @@ final class DocFieldProcessor extends Do
}
}
- final private Map<String, DocValuesConsumer> docValues = new HashMap<String, DocValuesConsumer>();
+ private static class DocValuesConsumerAndDocID {
+ public int docID;
+ final DocValuesConsumer docValuesConsumer;
+
+ public DocValuesConsumerAndDocID(DocValuesConsumer docValuesConsumer) {
+ this.docValuesConsumer = docValuesConsumer;
+ }
+ }
+
+ final private Map<String, DocValuesConsumerAndDocID> docValues = new HashMap<String, DocValuesConsumerAndDocID>();
final private Map<Integer, PerDocConsumer> perDocConsumers = new HashMap<Integer, PerDocConsumer>();
DocValuesConsumer docValuesConsumer(DocState docState, FieldInfo fieldInfo)
throws IOException {
- DocValuesConsumer docValuesConsumer = docValues.get(fieldInfo.name);
- if (docValuesConsumer != null) {
- return docValuesConsumer;
+ DocValuesConsumerAndDocID docValuesConsumerAndDocID = docValues.get(fieldInfo.name);
+ if (docValuesConsumerAndDocID != null) {
+ if (docState.docID == docValuesConsumerAndDocID.docID) {
+ throw new IllegalArgumentException("IndexDocValuesField \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed, per field)");
+ }
+ assert docValuesConsumerAndDocID.docID < docState.docID;
+ docValuesConsumerAndDocID.docID = docState.docID;
+ return docValuesConsumerAndDocID.docValuesConsumer;
}
PerDocConsumer perDocConsumer = perDocConsumers.get(fieldInfo.getCodecId());
if (perDocConsumer == null) {
@@ -316,6 +330,7 @@ final class DocFieldProcessor extends Do
perDocConsumers.put(Integer.valueOf(fieldInfo.getCodecId()), perDocConsumer);
}
boolean success = false;
+ DocValuesConsumer docValuesConsumer = null;
try {
docValuesConsumer = perDocConsumer.addValuesField(fieldInfo);
fieldInfo.commitDocValues();
@@ -325,7 +340,10 @@ final class DocFieldProcessor extends Do
fieldInfo.revertUncommitted();
}
}
- docValues.put(fieldInfo.name, docValuesConsumer);
+
+ docValuesConsumerAndDocID = new DocValuesConsumerAndDocID(docValuesConsumer);
+ docValuesConsumerAndDocID.docID = docState.docID;
+ docValues.put(fieldInfo.name, docValuesConsumerAndDocID);
return docValuesConsumer;
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java Thu Sep 29 21:49:45 2011
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.PerDocValues;
@@ -199,11 +200,45 @@ public abstract class IndexReader implem
* references.
*
* @see #decRef
+ * @see #tryIncRef
*/
public void incRef() {
ensureOpen();
refCount.incrementAndGet();
}
+
+ /**
+ * Expert: increments the refCount of this IndexReader
+ * instance only if the IndexReader has not been closed yet
+ * and returns <code>true</code> iff the refCount was
+ * successfully incremented, otherwise <code>false</code>.
+ * If this method returns <code>false</code> the reader is either
+ * already closed or is currently been closed. Either way this
+ * reader instance shouldn't be used by an application unless
+ * <code>true</code> is returned.
+ * <p>
+ * RefCounts are used to determine when a
+ * reader can be closed safely, i.e. as soon as there are
+ * no more references. Be sure to always call a
+ * corresponding {@link #decRef}, in a finally clause;
+ * otherwise the reader may never be closed. Note that
+ * {@link #close} simply calls decRef(), which means that
+ * the IndexReader will not really be closed until {@link
+ * #decRef} has been called for all outstanding
+ * references.
+ *
+ * @see #decRef
+ * @see #incRef
+ */
+ public boolean tryIncRef() {
+ int count;
+ while ((count = refCount.get()) > 0) {
+ if(refCount.compareAndSet(count, count+1)) {
+ return true;
+ }
+ }
+ return false;
+ }
/** {@inheritDoc} */
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java Thu Sep 29 21:49:45 2011
@@ -188,12 +188,12 @@ public class PersistentSnapshotDeletionP
Document d = new Document();
FieldType ft = new FieldType();
ft.setStored(true);
- d.add(new Field(SNAPSHOTS_ID, ft, ""));
+ d.add(new Field(SNAPSHOTS_ID, "", ft));
for (Entry<String, String> e : super.getSnapshots().entrySet()) {
- d.add(new Field(e.getKey(), ft, e.getValue()));
+ d.add(new Field(e.getKey(), e.getValue(), ft));
}
if (id != null) {
- d.add(new Field(id, ft, segment));
+ d.add(new Field(id, segment, ft));
}
writer.addDocument(d);
writer.commit();
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/StoredFieldVisitor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/StoredFieldVisitor.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/StoredFieldVisitor.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/StoredFieldVisitor.java Thu Sep 29 21:49:45 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.store.IndexInput;
/**
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java Thu Sep 29 21:49:45 2011
@@ -154,7 +154,7 @@ public class DefaultDocValuesProducer ex
case FIXED_INTS_64:
case FIXED_INTS_8:
case VAR_INTS:
- return Ints.getValues(dir, id, docCount, context);
+ return Ints.getValues(dir, id, docCount, type, context);
case FLOAT_32:
return Floats.getValues(dir, id, docCount, context);
case FLOAT_64:
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Bytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Bytes.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Bytes.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Bytes.java Thu Sep 29 21:49:45 2011
@@ -27,16 +27,25 @@ import org.apache.lucene.index.IndexFile
import org.apache.lucene.index.values.IndexDocValues.SortedSource;
import org.apache.lucene.index.values.IndexDocValues.Source;
import org.apache.lucene.index.values.IndexDocValues.SourceEnum;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PagedBytes;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.ByteBlockPool.Allocator;
+import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;
+import org.apache.lucene.util.BytesRefHash.TrackingDirectBytesStartArray;
+import org.apache.lucene.util.packed.PackedInts;
/**
* Provides concrete Writer/Reader implementations for <tt>byte[]</tt> value per
@@ -185,16 +194,18 @@ public final class Bytes {
}
// TODO open up this API?
- static abstract class BytesBaseSource extends Source {
+ static abstract class BytesSourceBase extends Source {
private final PagedBytes pagedBytes;
+ private final ValueType type;
protected final IndexInput datIn;
protected final IndexInput idxIn;
protected final static int PAGED_BYTES_BITS = 15;
protected final PagedBytes.Reader data;
protected final long totalLengthInBytes;
+
- protected BytesBaseSource(IndexInput datIn, IndexInput idxIn,
- PagedBytes pagedBytes, long bytesToRead) throws IOException {
+ protected BytesSourceBase(IndexInput datIn, IndexInput idxIn,
+ PagedBytes pagedBytes, long bytesToRead, ValueType type) throws IOException {
assert bytesToRead <= datIn.length() : " file size is less than the expected size diff: "
+ (bytesToRead - datIn.length()) + " pos: " + datIn.getFilePointer();
this.datIn = datIn;
@@ -203,6 +214,7 @@ public final class Bytes {
this.pagedBytes.copy(datIn, bytesToRead);
data = pagedBytes.freeze(true);
this.idxIn = idxIn;
+ this.type = type;
}
public void close() throws IOException {
@@ -220,6 +232,17 @@ public final class Bytes {
}
}
}
+
+ @Override
+ public ValueType type() {
+ return type;
+ }
+
+
+ @Override
+ public int getValueCount() {
+ throw new UnsupportedOperationException();
+ }
/**
* Returns one greater than the largest possible document number.
@@ -245,18 +268,46 @@ public final class Bytes {
}
}
+
+ static abstract class DerefBytesSourceBase extends BytesSourceBase {
+ protected final PackedInts.Reader addresses;
+ public DerefBytesSourceBase(IndexInput datIn, IndexInput idxIn, long bytesToRead, ValueType type) throws IOException {
+ super(datIn, idxIn, new PagedBytes(PAGED_BYTES_BITS), bytesToRead, type);
+ addresses = PackedInts.getReader(idxIn);
+ }
+
+ @Override
+ public int getValueCount() {
+ return addresses.size();
+ }
+
+ @Override
+ protected int maxDoc() {
+ return addresses.size();
+ }
+
+ }
- static abstract class BytesBaseSortedSource extends SortedSource {
+ static abstract class BytesSortedSourceBase extends SortedSource {
+ private final PagedBytes pagedBytes;
+ private final Comparator<BytesRef> comp;
+ protected final PackedInts.Reader docToOrdIndex;
+ private final ValueType type;
+
protected final IndexInput datIn;
protected final IndexInput idxIn;
protected final BytesRef defaultValue = new BytesRef();
protected final static int PAGED_BYTES_BITS = 15;
- private final PagedBytes pagedBytes;
protected final PagedBytes.Reader data;
- private final Comparator<BytesRef> comp;
+
- protected BytesBaseSortedSource(IndexInput datIn, IndexInput idxIn,
- Comparator<BytesRef> comp, PagedBytes pagedBytes, long bytesToRead)
+ protected BytesSortedSourceBase(IndexInput datIn, IndexInput idxIn,
+ Comparator<BytesRef> comp, long bytesToRead, ValueType type) throws IOException {
+ this(datIn, idxIn, comp, new PagedBytes(PAGED_BYTES_BITS), bytesToRead, type);
+ }
+
+ protected BytesSortedSourceBase(IndexInput datIn, IndexInput idxIn,
+ Comparator<BytesRef> comp, PagedBytes pagedBytes, long bytesToRead,ValueType type)
throws IOException {
assert bytesToRead <= datIn.length() : " file size is less than the expected size diff: "
+ (bytesToRead - datIn.length()) + " pos: " + datIn.getFilePointer();
@@ -267,8 +318,15 @@ public final class Bytes {
this.idxIn = idxIn;
this.comp = comp == null ? BytesRef.getUTF8SortedAsUnicodeComparator()
: comp;
+ docToOrdIndex = PackedInts.getReader(idxIn);
+ this.type = type;
}
+
+ @Override
+ public int ord(int docID) {
+ return (int) docToOrdIndex.get(docID) -1;
+ }
@Override
public BytesRef getByOrd(int ord, BytesRef bytesRef) {
@@ -277,22 +335,15 @@ public final class Bytes {
}
protected void closeIndexInput() throws IOException {
- try {
- if (datIn != null) {
- datIn.close();
- }
- } finally {
- if (idxIn != null) {// if straight
- idxIn.close();
- }
- }
+ IOUtils.close(datIn, idxIn);
}
-
+
/**
* Returns the largest doc id + 1 in this doc values source
*/
- protected abstract int maxDoc();
-
+ public int maxDoc() {
+ return docToOrdIndex.size();
+ }
/**
* Copies the value for the given ord to the given {@link BytesRef} and
* returns it.
@@ -336,6 +387,11 @@ public final class Bytes {
}
};
}
+
+ @Override
+ public ValueType type() {
+ return type;
+ }
}
// TODO: open up this API?!
@@ -359,7 +415,7 @@ public final class Bytes {
this.context = context;
}
- protected IndexOutput getDataOut() throws IOException {
+ protected IndexOutput getOrCreateDataOut() throws IOException {
if (datOut == null) {
boolean success = false;
try {
@@ -375,8 +431,16 @@ public final class Bytes {
}
return datOut;
}
+
+ protected IndexOutput getIndexOut() {
+ return idxOut;
+ }
+
+ protected IndexOutput getDataOut() {
+ return datOut;
+ }
- protected IndexOutput getIndexOut() throws IOException {
+ protected IndexOutput getOrCreateIndexOut() throws IOException {
boolean success = false;
try {
if (idxOut == null) {
@@ -503,5 +567,223 @@ public final class Bytes {
}
}
}
+
+ static abstract class DerefBytesWriterBase extends BytesWriterBase {
+ protected int size = -1;
+ protected int[] docToEntry;
+ protected final BytesRefHash hash;
+
+ protected DerefBytesWriterBase(Directory dir, String id, String codecName,
+ int codecVersion, Counter bytesUsed, IOContext context)
+ throws IOException {
+ this(dir, id, codecName, codecVersion, new DirectTrackingAllocator(
+ ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context);
+ }
+
+ protected DerefBytesWriterBase(Directory dir, String id, String codecName, int codecVersion, Allocator allocator,
+ Counter bytesUsed, IOContext context) throws IOException {
+ super(dir, id, codecName, codecVersion, bytesUsed, context);
+ hash = new BytesRefHash(new ByteBlockPool(allocator),
+ BytesRefHash.DEFAULT_CAPACITY, new TrackingDirectBytesStartArray(
+ BytesRefHash.DEFAULT_CAPACITY, bytesUsed));
+ docToEntry = new int[1];
+ bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT);
+ }
+
+ protected static int writePrefixLength(DataOutput datOut, BytesRef bytes)
+ throws IOException {
+ if (bytes.length < 128) {
+ datOut.writeByte((byte) bytes.length);
+ return 1;
+ } else {
+ datOut.writeByte((byte) (0x80 | (bytes.length >> 8)));
+ datOut.writeByte((byte) (bytes.length & 0xff));
+ return 2;
+ }
+ }
+
+ @Override
+ public void add(int docID, BytesRef bytes) throws IOException {
+ if (bytes.length == 0) { // default value - skip it
+ return;
+ }
+ checkSize(bytes);
+ int ord = hash.add(bytes);
+ if (ord < 0) {
+ ord = (-ord) - 1;
+ }
+ if (docID >= docToEntry.length) {
+ final int size = docToEntry.length;
+ docToEntry = ArrayUtil.grow(docToEntry, 1 + docID);
+ bytesUsed.addAndGet((docToEntry.length - size)
+ * RamUsageEstimator.NUM_BYTES_INT);
+ }
+ docToEntry[docID] = 1 + ord;
+ }
+
+ protected void checkSize(BytesRef bytes) {
+ if (size == -1) {
+ size = bytes.length;
+ } else if (bytes.length != size) {
+ throw new IllegalArgumentException("expected bytes size=" + size
+ + " but got " + bytes.length);
+ }
+ }
+
+ // Important that we get docCount, in case there were
+ // some last docs that we didn't see
+ @Override
+ public void finish(int docCount) throws IOException {
+ boolean success = false;
+ try {
+ finishInternal(docCount);
+ success = true;
+ } finally {
+ releaseResources();
+ if (success) {
+ IOUtils.close(getIndexOut(), getDataOut());
+ } else {
+ IOUtils.closeWhileHandlingException(getIndexOut(), getDataOut());
+ }
+
+ }
+ }
+
+ protected abstract void finishInternal(int docCount) throws IOException;
+
+ protected void releaseResources() {
+ hash.close();
+ bytesUsed
+ .addAndGet((-docToEntry.length) * RamUsageEstimator.NUM_BYTES_INT);
+ docToEntry = null;
+ }
+
+ protected void writeIndex(IndexOutput idxOut, int docCount,
+ long maxValue, int[] toEntry) throws IOException {
+ writeIndex(idxOut, docCount, maxValue, (int[])null, toEntry);
+ }
+
+ protected void writeIndex(IndexOutput idxOut, int docCount,
+ long maxValue, int[] addresses, int[] toEntry) throws IOException {
+ final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount,
+ PackedInts.bitsRequired(maxValue));
+ final int limit = docCount > docToEntry.length ? docToEntry.length
+ : docCount;
+ assert toEntry.length >= limit -1;
+ if (addresses != null) {
+ for (int i = 0; i < limit; i++) {
+ assert addresses[toEntry[i]] >= 0;
+ w.add(addresses[toEntry[i]]);
+ }
+ } else {
+ for (int i = 0; i < limit; i++) {
+ assert toEntry[i] >= 0;
+ w.add(toEntry[i]);
+ }
+ }
+ for (int i = limit; i < docCount; i++) {
+ w.add(0);
+ }
+ w.finish();
+ }
+
+ protected void writeIndex(IndexOutput idxOut, int docCount,
+ long maxValue, long[] addresses, int[] toEntry) throws IOException {
+ final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount,
+ PackedInts.bitsRequired(maxValue));
+ final int limit = docCount > docToEntry.length ? docToEntry.length
+ : docCount;
+ assert toEntry.length >= limit -1;
+ if (addresses != null) {
+ for (int i = 0; i < limit; i++) {
+ assert addresses[toEntry[i]] >= 0;
+ w.add(addresses[toEntry[i]]);
+ }
+ } else {
+ for (int i = 0; i < limit; i++) {
+ assert toEntry[i] >= 0;
+ w.add(toEntry[i]);
+ }
+ }
+ for (int i = limit; i < docCount; i++) {
+ w.add(0);
+ }
+ w.finish();
+ }
+
+ }
+
+ abstract static class DerefBytesEnumBase extends ValuesEnum {
+ private final PackedInts.ReaderIterator idx;
+ private final int valueCount;
+ private int pos = -1;
+ protected final IndexInput datIn;
+ protected final long fp;
+ protected final int size;
+
+ protected DerefBytesEnumBase(AttributeSource source, IndexInput datIn,
+ IndexInput idxIn, int size, ValueType enumType) throws IOException {
+ super(source, enumType);
+ this.datIn = datIn;
+ this.size = size;
+ idx = PackedInts.getReaderIterator(idxIn);
+ fp = datIn.getFilePointer();
+ if (size > 0) {
+ bytesRef.grow(this.size);
+ bytesRef.length = this.size;
+ }
+ bytesRef.offset = 0;
+ valueCount = idx.size();
+ }
+
+ protected void copyFrom(ValuesEnum valuesEnum) {
+ bytesRef = valuesEnum.bytesRef;
+ if (bytesRef.bytes.length < size) {
+ bytesRef.grow(size);
+ }
+ bytesRef.length = size;
+ bytesRef.offset = 0;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target < valueCount) {
+ long address;
+ while ((address = idx.advance(target)) == 0) {
+ if (++target >= valueCount) {
+ return pos = NO_MORE_DOCS;
+ }
+ }
+ pos = idx.ord();
+ fill(address, bytesRef);
+ return pos;
+ }
+ return pos = NO_MORE_DOCS;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (pos >= valueCount) {
+ return pos = NO_MORE_DOCS;
+ }
+ return advance(pos + 1);
+ }
+
+ public void close() throws IOException {
+ try {
+ datIn.close();
+ } finally {
+ idx.close();
+ }
+ }
+
+ protected abstract void fill(long address, BytesRef ref) throws IOException;
+
+ @Override
+ public int docID() {
+ return pos;
+ }
+
+ }
}
\ No newline at end of file
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java Thu Sep 29 21:49:45 2011
@@ -19,26 +19,17 @@ package org.apache.lucene.index.values;
import java.io.IOException;
-import org.apache.lucene.index.values.Bytes.BytesBaseSource;
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
-import org.apache.lucene.index.values.Bytes.BytesWriterBase;
+import org.apache.lucene.index.values.Bytes.DerefBytesSourceBase;
+import org.apache.lucene.index.values.Bytes.DerefBytesEnumBase;
+import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeSource;
-import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.Counter;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.PagedBytes;
-import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.lucene.util.ByteBlockPool.Allocator;
-import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;
-import org.apache.lucene.util.BytesRefHash.TrackingDirectBytesStartArray;
-import org.apache.lucene.util.packed.PackedInts;
// Stores fixed-length byte[] by deref, ie when two docs
// have the same value, they store only 1 byte[]
@@ -51,135 +42,55 @@ class FixedDerefBytesImpl {
static final int VERSION_START = 0;
static final int VERSION_CURRENT = VERSION_START;
- static class Writer extends BytesWriterBase {
- private int size = -1;
- private int[] docToID;
- private final BytesRefHash hash;
+ public static class Writer extends DerefBytesWriterBase {
public Writer(Directory dir, String id, Counter bytesUsed, IOContext context)
throws IOException {
- this(dir, id, new DirectTrackingAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed),
- bytesUsed, context);
- }
-
- public Writer(Directory dir, String id, Allocator allocator,
- Counter bytesUsed, IOContext context) throws IOException {
super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
- hash = new BytesRefHash(new ByteBlockPool(allocator),
- BytesRefHash.DEFAULT_CAPACITY, new TrackingDirectBytesStartArray(
- BytesRefHash.DEFAULT_CAPACITY, bytesUsed));
- docToID = new int[1];
- bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT);
- }
-
- @Override
- public void add(int docID, BytesRef bytes) throws IOException {
- if (bytes.length == 0) // default value - skip it
- return;
- if (size == -1) {
- size = bytes.length;
- } else if (bytes.length != size) {
- throw new IllegalArgumentException("expected bytes size=" + size
- + " but got " + bytes.length);
- }
- int ord = hash.add(bytes);
- if (ord < 0) {
- ord = (-ord) - 1;
- }
- if (docID >= docToID.length) {
- final int size = docToID.length;
- docToID = ArrayUtil.grow(docToID, 1 + docID);
- bytesUsed.addAndGet((docToID.length - size)
- * RamUsageEstimator.NUM_BYTES_INT);
- }
- docToID[docID] = 1 + ord;
}
- // Important that we get docCount, in case there were
- // some last docs that we didn't see
@Override
- public void finish(int docCount) throws IOException {
- boolean success = false;
+ protected void finishInternal(int docCount) throws IOException {
final int numValues = hash.size();
- final IndexOutput datOut = getDataOut();
- try {
- datOut.writeInt(size);
- if (size != -1) {
- final BytesRef bytesRef = new BytesRef(size);
- for (int i = 0; i < numValues; i++) {
- hash.get(i, bytesRef);
- datOut.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
- }
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(datOut);
- } else {
- IOUtils.closeWhileHandlingException(datOut);
+ final IndexOutput datOut = getOrCreateDataOut();
+ datOut.writeInt(size);
+ if (size != -1) {
+ final BytesRef bytesRef = new BytesRef(size);
+ for (int i = 0; i < numValues; i++) {
+ hash.get(i, bytesRef);
+ datOut.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
}
- hash.close();
- }
- success = false;
- final IndexOutput idxOut = getIndexOut();
- try {
- final int count = 1 + numValues;
- idxOut.writeInt(count - 1);
- // write index
- final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount,
- PackedInts.bitsRequired(count - 1));
- final int limit = docCount > docToID.length ? docToID.length : docCount;
- for (int i = 0; i < limit; i++) {
- w.add(docToID[i]);
- }
- // fill up remaining doc with zeros
- for (int i = limit; i < docCount; i++) {
- w.add(0);
- }
- w.finish();
- success = true;
- } finally {
- if (success) {
- IOUtils.close(idxOut);
- } else {
- IOUtils.closeWhileHandlingException(idxOut);
- }
- bytesUsed
- .addAndGet((-docToID.length) * RamUsageEstimator.NUM_BYTES_INT);
- docToID = null;
}
+ final IndexOutput idxOut = getOrCreateIndexOut();
+ idxOut.writeInt(numValues);
+ writeIndex(idxOut, docCount, numValues, docToEntry);
}
}
public static class Reader extends BytesReaderBase {
private final int size;
-
+ private final int numValuesStored;
Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
super(dir, id, CODEC_NAME, VERSION_START, true, context);
size = datIn.readInt();
+ numValuesStored = idxIn.readInt();
}
@Override
public Source load() throws IOException {
- final IndexInput index = cloneIndex();
- return new Source(cloneData(), index, size, index.readInt());
+ return new Source(cloneData(), cloneIndex(), size, numValuesStored);
}
- private static class Source extends BytesBaseSource {
- private final PackedInts.Reader index;
+ private static final class Source extends DerefBytesSourceBase {
private final int size;
- private final int numValues;
- protected Source(IndexInput datIn, IndexInput idxIn, int size,
- int numValues) throws IOException {
- super(datIn, idxIn, new PagedBytes(PAGED_BYTES_BITS), size * numValues);
+ protected Source(IndexInput datIn, IndexInput idxIn, int size, long numValues) throws IOException {
+ super(datIn, idxIn, size * numValues, ValueType.BYTES_FIXED_DEREF);
this.size = size;
- this.numValues = numValues;
- index = PackedInts.getReader(idxIn);
}
@Override
public BytesRef getBytes(int docID, BytesRef bytesRef) {
- final int id = (int) index.get(docID);
+ final int id = (int) addresses.get(docID);
if (id == 0) {
bytesRef.length = 0;
return bytesRef;
@@ -187,95 +98,18 @@ class FixedDerefBytesImpl {
return data.fillSlice(bytesRef, ((id - 1) * size), size);
}
- @Override
- public int getValueCount() {
- return numValues;
- }
-
- @Override
- public ValueType type() {
- return ValueType.BYTES_FIXED_DEREF;
- }
-
- @Override
- protected int maxDoc() {
- return index.size();
- }
}
@Override
public ValuesEnum getEnum(AttributeSource source) throws IOException {
- return new DerefBytesEnum(source, cloneData(), cloneIndex(), size);
+ return new DerefBytesEnum(source, cloneData(), cloneIndex(), size);
}
- static class DerefBytesEnum extends ValuesEnum {
- protected final IndexInput datIn;
- private final PackedInts.ReaderIterator idx;
- protected final long fp;
- private final int size;
- private final int valueCount;
- private int pos = -1;
+ final static class DerefBytesEnum extends DerefBytesEnumBase {
public DerefBytesEnum(AttributeSource source, IndexInput datIn,
IndexInput idxIn, int size) throws IOException {
- this(source, datIn, idxIn, size, ValueType.BYTES_FIXED_DEREF);
- }
-
- protected DerefBytesEnum(AttributeSource source, IndexInput datIn,
- IndexInput idxIn, int size, ValueType enumType) throws IOException {
- super(source, enumType);
- this.datIn = datIn;
- this.size = size;
- idxIn.readInt();// read valueCount
- idx = PackedInts.getReaderIterator(idxIn);
- fp = datIn.getFilePointer();
- if (size > 0) {
- bytesRef.grow(this.size);
- bytesRef.length = this.size;
- }
- bytesRef.offset = 0;
- valueCount = idx.size();
- }
-
- protected void copyFrom(ValuesEnum valuesEnum) {
- bytesRef = valuesEnum.bytesRef;
- if (bytesRef.bytes.length < size) {
- bytesRef.grow(size);
- }
- bytesRef.length = size;
- bytesRef.offset = 0;
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (target < valueCount) {
- long address;
- while ((address = idx.advance(target)) == 0) {
- if (++target >= valueCount) {
- return pos = NO_MORE_DOCS;
- }
- }
- pos = idx.ord();
- fill(address, bytesRef);
- return pos;
- }
- return pos = NO_MORE_DOCS;
- }
-
- @Override
- public int nextDoc() throws IOException {
- if (pos >= valueCount) {
- return pos = NO_MORE_DOCS;
- }
- return advance(pos + 1);
- }
-
- public void close() throws IOException {
- try {
- datIn.close();
- } finally {
- idx.close();
- }
+ super(source, datIn, idxIn, size, ValueType.BYTES_FIXED_DEREF);
}
protected void fill(long address, BytesRef ref) throws IOException {
@@ -284,12 +118,6 @@ class FixedDerefBytesImpl {
ref.length = size;
ref.offset = 0;
}
-
- @Override
- public int docID() {
- return pos;
- }
-
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java Thu Sep 29 21:49:45 2011
@@ -20,28 +20,17 @@ package org.apache.lucene.index.values;
import java.io.IOException;
import java.util.Comparator;
-import org.apache.lucene.index.values.Bytes.BytesBaseSortedSource;
+import org.apache.lucene.index.values.Bytes.BytesSortedSourceBase;
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
-import org.apache.lucene.index.values.Bytes.BytesWriterBase;
+import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase;
import org.apache.lucene.index.values.FixedDerefBytesImpl.Reader.DerefBytesEnum;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeSource;
-import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefHash;
-import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.Counter;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.PagedBytes;
-import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.lucene.util.ByteBlockPool.Allocator;
-import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;
-import org.apache.lucene.util.BytesRefHash.TrackingDirectBytesStartArray;
-import org.apache.lucene.util.packed.PackedInts;
// Stores fixed-length byte[] by deref, ie when two docs
// have the same value, they store only 1 byte[]
@@ -55,132 +44,49 @@ class FixedSortedBytesImpl {
static final int VERSION_START = 0;
static final int VERSION_CURRENT = VERSION_START;
- static class Writer extends BytesWriterBase {
- private int size = -1;
- private int[] docToEntry;
+ static class Writer extends DerefBytesWriterBase {
private final Comparator<BytesRef> comp;
- private final BytesRefHash hash;
public Writer(Directory dir, String id, Comparator<BytesRef> comp,
Counter bytesUsed, IOContext context) throws IOException {
- this(dir, id, comp, new DirectTrackingAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed),
- bytesUsed, context);
- }
-
- public Writer(Directory dir, String id, Comparator<BytesRef> comp,
- Allocator allocator, Counter bytesUsed, IOContext context) throws IOException {
super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
- ByteBlockPool pool = new ByteBlockPool(allocator);
- hash = new BytesRefHash(pool, BytesRefHash.DEFAULT_CAPACITY,
- new TrackingDirectBytesStartArray(BytesRefHash.DEFAULT_CAPACITY,
- bytesUsed));
- docToEntry = new int[1];
- bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT);
this.comp = comp;
}
- @Override
- public void add(int docID, BytesRef bytes) throws IOException {
- if (bytes.length == 0)
- return; // default - skip it
- if (size == -1) {
- size = bytes.length;
- } else if (bytes.length != size) {
- throw new IllegalArgumentException("expected bytes size=" + size
- + " but got " + bytes.length);
- }
- if (docID >= docToEntry.length) {
- final int[] newArray = new int[ArrayUtil.oversize(1 + docID,
- RamUsageEstimator.NUM_BYTES_INT)];
- System.arraycopy(docToEntry, 0, newArray, 0, docToEntry.length);
- bytesUsed.addAndGet((newArray.length - docToEntry.length)
- * RamUsageEstimator.NUM_BYTES_INT);
- docToEntry = newArray;
- }
- int e = hash.add(bytes);
- docToEntry[docID] = 1 + (e < 0 ? (-e) - 1 : e);
- }
-
// Important that we get docCount, in case there were
// some last docs that we didn't see
@Override
- public void finish(int docCount) throws IOException {
- final IndexOutput datOut = getDataOut();
- boolean success = false;
+ public void finishInternal(int docCount) throws IOException {
+ final IndexOutput datOut = getOrCreateDataOut();
final int count = hash.size();
- final int[] address = new int[count];
-
- try {
- datOut.writeInt(size);
- if (size != -1) {
- final int[] sortedEntries = hash.sort(comp);
- // first dump bytes data, recording address as we go
- final BytesRef bytesRef = new BytesRef(size);
- for (int i = 0; i < count; i++) {
- final int e = sortedEntries[i];
- final BytesRef bytes = hash.get(e, bytesRef);
- assert bytes.length == size;
- datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length);
- address[e] = 1 + i;
- }
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(datOut);
- } else {
- IOUtils.closeWhileHandlingException(datOut);
- }
- hash.close();
- }
- final IndexOutput idxOut = getIndexOut();
- success = false;
- try {
- idxOut.writeInt(count);
- // next write index
- final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount,
- PackedInts.bitsRequired(count));
- final int limit;
- if (docCount > docToEntry.length) {
- limit = docToEntry.length;
- } else {
- limit = docCount;
- }
- for (int i = 0; i < limit; i++) {
- final int e = docToEntry[i];
- if (e == 0) {
- // null is encoded as zero
- w.add(0);
- } else {
- assert e > 0 && e <= count : "index must 0 > && <= " + count
- + " was: " + e;
- w.add(address[e - 1]);
- }
- }
-
- for (int i = limit; i < docCount; i++) {
- w.add(0);
- }
- w.finish();
- } finally {
- if (success) {
- IOUtils.close(idxOut);
- } else {
- IOUtils.closeWhileHandlingException(idxOut);
- }
- bytesUsed.addAndGet((-docToEntry.length)
- * RamUsageEstimator.NUM_BYTES_INT);
- docToEntry = null;
- }
+ final int[] address = new int[count+1]; // addr 0 is default values
+ datOut.writeInt(size);
+ if (size != -1) {
+ final int[] sortedEntries = hash.sort(comp);
+ // first dump bytes data, recording address as we go
+ final BytesRef bytesRef = new BytesRef(size);
+ for (int i = 0; i < count; i++) {
+ final int e = sortedEntries[i];
+ final BytesRef bytes = hash.get(e, bytesRef);
+ assert bytes.length == size;
+ datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length);
+ address[e + 1] = 1 + i;
+ }
+ }
+ final IndexOutput idxOut = getOrCreateIndexOut();
+ idxOut.writeInt(count);
+ writeIndex(idxOut, docCount, count, address, docToEntry);
}
}
public static class Reader extends BytesReaderBase {
private final int size;
+ private final int numValuesStored;
public Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
super(dir, id, CODEC_NAME, VERSION_START, true, context);
size = datIn.readInt();
+ numValuesStored = idxIn.readInt();
}
@Override
@@ -192,58 +98,35 @@ class FixedSortedBytesImpl {
@Override
public SortedSource loadSorted(Comparator<BytesRef> comp)
throws IOException {
- final IndexInput idxInput = cloneIndex();
- final IndexInput datInput = cloneData();
- datInput.seek(CodecUtil.headerLength(CODEC_NAME) + 4);
- idxInput.seek(CodecUtil.headerLength(CODEC_NAME));
- return new Source(datInput, idxInput, size, idxInput.readInt(), comp);
+ return new Source(cloneData(), cloneIndex(), size, numValuesStored, comp);
}
- private static class Source extends BytesBaseSortedSource {
-
- private final PackedInts.Reader index;
- private final int numValue;
+ private static class Source extends BytesSortedSourceBase {
+ private final int valueCount;
private final int size;
public Source(IndexInput datIn, IndexInput idxIn, int size,
int numValues, Comparator<BytesRef> comp) throws IOException {
- super(datIn, idxIn, comp, new PagedBytes(PAGED_BYTES_BITS), size
- * numValues);
+ super(datIn, idxIn, comp, size * numValues, ValueType.BYTES_FIXED_SORTED);
this.size = size;
- this.numValue = numValues;
- index = PackedInts.getReader(idxIn);
+ this.valueCount = numValues;
closeIndexInput();
}
@Override
- public int ord(int docID) {
- return (int) index.get(docID) -1;
- }
-
- @Override
public int getByValue(BytesRef bytes, BytesRef tmpRef) {
- return binarySearch(bytes, tmpRef, 0, numValue - 1);
+ return binarySearch(bytes, tmpRef, 0, valueCount - 1);
}
@Override
public int getValueCount() {
- return numValue;
+ return valueCount;
}
@Override
protected BytesRef deref(int ord, BytesRef bytesRef) {
return data.fillSlice(bytesRef, (ord * size), size);
}
-
- @Override
- public ValueType type() {
- return ValueType.BYTES_FIXED_SORTED;
- }
-
- @Override
- protected int maxDoc() {
- return index.size();
- }
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java Thu Sep 29 21:49:45 2011
@@ -21,7 +21,7 @@ import static org.apache.lucene.util.Byt
import java.io.IOException;
-import org.apache.lucene.index.values.Bytes.BytesBaseSource;
+import org.apache.lucene.index.values.Bytes.BytesSourceBase;
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
import org.apache.lucene.index.values.Bytes.BytesWriterBase;
import org.apache.lucene.store.Directory;
@@ -46,26 +46,24 @@ class FixedStraightBytesImpl {
static final String CODEC_NAME = "FixedStraightBytes";
static final int VERSION_START = 0;
static final int VERSION_CURRENT = VERSION_START;
-
- static class Writer extends BytesWriterBase {
- private int size = -1;
+
+ static abstract class FixedBytesWriterBase extends BytesWriterBase {
+ protected int lastDocID = -1;
// start at -1 if the first added value is > 0
- private int lastDocID = -1;
+ protected int size = -1;
+ private final int byteBlockSize = BYTE_BLOCK_SIZE;
private final ByteBlockPool pool;
- private boolean merge;
- private final int byteBlockSize;
- private IndexOutput datOut;
- public Writer(Directory dir, String id, Counter bytesUsed, IOContext context) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
+ protected FixedBytesWriterBase(Directory dir, String id, String codecName,
+ int version, Counter bytesUsed, IOContext context) throws IOException {
+ super(dir, id, codecName, version, bytesUsed, context);
pool = new ByteBlockPool(new DirectTrackingAllocator(bytesUsed));
- byteBlockSize = BYTE_BLOCK_SIZE;
}
-
+
@Override
public void add(int docID, BytesRef bytes) throws IOException {
assert lastDocID < docID;
- assert !merge;
+
if (size == -1) {
if (bytes.length > BYTE_BLOCK_SIZE) {
throw new IllegalArgumentException("bytes arrays > " + Short.MAX_VALUE + " are not supported");
@@ -84,7 +82,6 @@ class FixedStraightBytesImpl {
}
private final void advancePool(int docID) {
- assert !merge;
long numBytes = (docID - (lastDocID+1))*size;
while(numBytes > 0) {
if (numBytes + pool.byteUpto < byteBlockSize) {
@@ -97,14 +94,50 @@ class FixedStraightBytesImpl {
}
assert numBytes == 0;
}
+
+ protected void set(BytesRef ref, int docId) {
+ assert BYTE_BLOCK_SIZE % size == 0 : "BYTE_BLOCK_SIZE ("+ BYTE_BLOCK_SIZE + ") must be a multiple of the size: " + size;
+ ref.offset = docId*size;
+ ref.length = size;
+ pool.deref(ref);
+ }
+
+ protected void resetPool() {
+ pool.dropBuffersAndReset();
+ }
+
+ protected void writeData(IndexOutput out) throws IOException {
+ pool.writePool(out);
+ }
+
+ protected void writeZeros(int num, IndexOutput out) throws IOException {
+ final byte[] zeros = new byte[size];
+ for (int i = 0; i < num; i++) {
+ out.writeBytes(zeros, zeros.length);
+ }
+ }
+ }
+
+ static class Writer extends FixedBytesWriterBase {
+ private boolean merge;
+ private IndexOutput datOut;
+
+ public Writer(Directory dir, String id, Counter bytesUsed, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
+ }
+
+ public Writer(Directory dir, String id, String codecName, int version, Counter bytesUsed, IOContext context) throws IOException {
+ super(dir, id, codecName, version, bytesUsed, context);
+ }
+
@Override
protected void merge(MergeState state) throws IOException {
merge = true;
- datOut = getDataOut();
+ datOut = getOrCreateDataOut();
boolean success = false;
try {
- if (state.liveDocs == null && state.reader instanceof Reader) {
+ if (state.liveDocs == null && state.reader instanceof Reader ) {
Reader reader = (Reader) state.reader;
final int maxDocs = reader.maxDoc;
if (maxDocs == 0) {
@@ -113,7 +146,10 @@ class FixedStraightBytesImpl {
if (size == -1) {
size = reader.size;
datOut.writeInt(size);
- }
+ } else if (size != reader.size) {
+ throw new IllegalArgumentException("expected bytes size=" + size
+ + " but got " + reader.size);
+ }
if (lastDocID+1 < state.docBase) {
fill(datOut, state.docBase);
lastDocID = state.docBase-1;
@@ -137,7 +173,7 @@ class FixedStraightBytesImpl {
}
}
}
-
+
@Override
protected void mergeDoc(int docID) throws IOException {
assert lastDocID < docID;
@@ -158,11 +194,7 @@ class FixedStraightBytesImpl {
// Fills up to but not including this docID
private void fill(IndexOutput datOut, int docID) throws IOException {
assert size >= 0;
- final long numBytes = (docID - (lastDocID+1))*size;
- final byte zero = 0;
- for (long i = 0; i < numBytes; i++) {
- datOut.writeByte(zero);
- }
+ writeZeros((docID - (lastDocID+1)), datOut);
}
@Override
@@ -172,12 +204,12 @@ class FixedStraightBytesImpl {
if (!merge) {
// indexing path - no disk IO until here
assert datOut == null;
- datOut = getDataOut();
+ datOut = getOrCreateDataOut();
if (size == -1) {
datOut.writeInt(0);
} else {
datOut.writeInt(size);
- pool.writePool(datOut);
+ writeData(datOut);
}
if (lastDocID + 1 < docCount) {
fill(datOut, docCount);
@@ -193,7 +225,7 @@ class FixedStraightBytesImpl {
}
success = true;
} finally {
- pool.dropBuffersAndReset();
+ resetPool();
if (success) {
IOUtils.close(datOut);
} else {
@@ -201,14 +233,19 @@ class FixedStraightBytesImpl {
}
}
}
+
}
public static class Reader extends BytesReaderBase {
- private final int size;
- private final int maxDoc;
-
+ protected final int size;
+ protected final int maxDoc;
+
Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_START, false, context);
+ this(dir, id, CODEC_NAME, VERSION_CURRENT, maxDoc, context);
+ }
+
+ protected Reader(Directory dir, String id, String codec, int version, int maxDoc, IOContext context) throws IOException {
+ super(dir, id, codec, version, false, context);
size = datIn.readInt();
this.maxDoc = maxDoc;
}
@@ -271,13 +308,13 @@ class FixedStraightBytesImpl {
}
- private static class StraightBytesSource extends BytesBaseSource {
+ private final static class StraightBytesSource extends BytesSourceBase {
private final int size;
private final int maxDoc;
public StraightBytesSource(IndexInput datIn, int size, int maxDoc)
throws IOException {
- super(datIn, null, new PagedBytes(PAGED_BYTES_BITS), size * maxDoc);
+ super(datIn, null, new PagedBytes(PAGED_BYTES_BITS), size * maxDoc, ValueType.BYTES_FIXED_STRAIGHT);
this.size = size;
this.maxDoc = maxDoc;
}
@@ -293,11 +330,6 @@ class FixedStraightBytesImpl {
}
@Override
- public ValueType type() {
- return ValueType.BYTES_FIXED_STRAIGHT;
- }
-
- @Override
protected int maxDoc() {
return maxDoc;
}
@@ -308,66 +340,68 @@ class FixedStraightBytesImpl {
return new FixedStraightBytesEnum(source, cloneData(), size, maxDoc);
}
- private static final class FixedStraightBytesEnum extends ValuesEnum {
- private final IndexInput datIn;
- private final int size;
- private final int maxDoc;
- private int pos = -1;
- private final long fp;
+
- public FixedStraightBytesEnum(AttributeSource source, IndexInput datIn,
- int size, int maxDoc) throws IOException {
- super(source, ValueType.BYTES_FIXED_STRAIGHT);
- this.datIn = datIn;
- this.size = size;
- this.maxDoc = maxDoc;
- bytesRef.grow(size);
- bytesRef.length = size;
- bytesRef.offset = 0;
- fp = datIn.getFilePointer();
- }
+ @Override
+ public ValueType type() {
+ return ValueType.BYTES_FIXED_STRAIGHT;
+ }
+ }
+
+ static class FixedStraightBytesEnum extends ValuesEnum {
+ private final IndexInput datIn;
+ private final int size;
+ private final int maxDoc;
+ private int pos = -1;
+ private final long fp;
- protected void copyFrom(ValuesEnum valuesEnum) {
- bytesRef = valuesEnum.bytesRef;
- if (bytesRef.bytes.length < size) {
- bytesRef.grow(size);
- }
- bytesRef.length = size;
- bytesRef.offset = 0;
- }
+ public FixedStraightBytesEnum(AttributeSource source, IndexInput datIn,
+ int size, int maxDoc) throws IOException {
+ super(source, ValueType.BYTES_FIXED_STRAIGHT);
+ this.datIn = datIn;
+ this.size = size;
+ this.maxDoc = maxDoc;
+ bytesRef.grow(size);
+ bytesRef.length = size;
+ bytesRef.offset = 0;
+ fp = datIn.getFilePointer();
+ }
- public void close() throws IOException {
- datIn.close();
+ protected void copyFrom(ValuesEnum valuesEnum) {
+ super.copyFrom(valuesEnum);
+ if (bytesRef.bytes.length < size) {
+ bytesRef.grow(size);
}
+ bytesRef.length = size;
+ bytesRef.offset = 0;
+ }
- @Override
- public int advance(int target) throws IOException {
- if (target >= maxDoc || size == 0) {
- return pos = NO_MORE_DOCS;
- }
- if ((target - 1) != pos) // pos inc == 1
- datIn.seek(fp + target * size);
- datIn.readBytes(bytesRef.bytes, 0, size);
- return pos = target;
- }
+ public void close() throws IOException {
+ datIn.close();
+ }
- @Override
- public int docID() {
- return pos;
- }
+ @Override
+ public int advance(int target) throws IOException {
+ if (target >= maxDoc || size == 0) {
+ return pos = NO_MORE_DOCS;
+ }
+ if ((target - 1) != pos) // pos inc == 1
+ datIn.seek(fp + target * size);
+ datIn.readBytes(bytesRef.bytes, 0, size);
+ return pos = target;
+ }
- @Override
- public int nextDoc() throws IOException {
- if (pos >= maxDoc) {
- return pos = NO_MORE_DOCS;
- }
- return advance(pos + 1);
- }
+ @Override
+ public int docID() {
+ return pos;
}
@Override
- public ValueType type() {
- return ValueType.BYTES_FIXED_STRAIGHT;
+ public int nextDoc() throws IOException {
+ if (pos >= maxDoc) {
+ return pos = NO_MORE_DOCS;
+ }
+ return advance(pos + 1);
}
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Floats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Floats.java?rev=1177441&r1=1177440&r2=1177441&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Floats.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Floats.java Thu Sep 29 21:49:45 2011
@@ -17,21 +17,14 @@ package org.apache.lucene.index.values;
* limitations under the License.
*/
import java.io.IOException;
-import java.util.Collection;
-import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.values.IndexDocValues.Source;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeSource;
-import org.apache.lucene.util.CodecUtil;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
-import org.apache.lucene.util.FloatsRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.RamUsageEstimator;
/**
* Exposes {@link Writer} and reader ({@link Source}) for 32 bit and 64 bit
@@ -43,11 +36,6 @@ import org.apache.lucene.util.RamUsageEs
* @lucene.experimental
*/
public class Floats {
- // TODO - add bulk copy where possible
- private static final String CODEC_NAME = "SimpleFloats";
- static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
- private static final byte[] DEFAULTS = new byte[] {0,0,0,0,0,0,0,0};
public static Writer getWriter(Directory dir, String id, int precisionBytes,
Counter bytesUsed, IOContext context) throws IOException {
@@ -55,493 +43,73 @@ public class Floats {
throw new IllegalArgumentException("precisionBytes must be 4 or 8; got "
+ precisionBytes);
}
- if (precisionBytes == 4) {
- return new Float4Writer(dir, id, bytesUsed, context);
- } else {
- return new Float8Writer(dir, id, bytesUsed, context);
- }
+ return new FloatsWriter(dir, id, bytesUsed, context, precisionBytes);
+
}
public static IndexDocValues getValues(Directory dir, String id, int maxDoc, IOContext context)
throws IOException {
return new FloatsReader(dir, id, maxDoc, context);
}
-
- abstract static class FloatsWriter extends Writer {
- private final String id;
- protected FloatsRef floatsRef;
- protected int lastDocId = -1;
- protected IndexOutput datOut;
- private final byte precision;
- private final Directory dir;
- private final IOContext context;
-
- protected FloatsWriter(Directory dir, String id, int precision,
- Counter bytesUsed, IOContext context) throws IOException {
- super(bytesUsed);
- this.id = id;
- this.precision = (byte) precision;
- this.dir = dir;
- this.context = context;
-
- }
-
- public long ramBytesUsed() {
- return 0;
+
+ final static class FloatsWriter extends FixedStraightBytesImpl.Writer {
+ private final int size;
+ public FloatsWriter(Directory dir, String id, Counter bytesUsed,
+ IOContext context, int size) throws IOException {
+ super(dir, id, bytesUsed, context);
+ this.bytesRef = new BytesRef(size);
+ this.size = size;
+ bytesRef.length = size;
}
- final void initDataOut() throws IOException {
- assert datOut == null;
- datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "",
- Writer.DATA_EXTENSION), context);
- boolean success = false;
- try {
- CodecUtil.writeHeader(datOut, CODEC_NAME, VERSION_CURRENT);
- assert datOut.getFilePointer() == CodecUtil.headerLength(CODEC_NAME);
- datOut.writeByte(this.precision);
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(datOut);
- }
+ public void add(int docID, double v) throws IOException {
+ if (size == 8) {
+ bytesRef.copy(Double.doubleToRawLongBits(v));
+ } else {
+ bytesRef.copy(Float.floatToRawIntBits((float)v));
}
+ add(docID, bytesRef);
}
-
- @Override
- protected void mergeDoc(int docID) throws IOException {
- add(docID, floatsRef.get());
- }
-
+
@Override
public void add(int docID, PerDocFieldValues docValues) throws IOException {
add(docID, docValues.getFloat());
}
-
- @Override
- protected void setNextEnum(ValuesEnum valuesEnum) {
- floatsRef = valuesEnum.getFloat();
- }
-
- protected final int fillDefault(int numValues) throws IOException {
- for (int i = 0; i < numValues; i++) {
- datOut.writeBytes(DEFAULTS, precision);
- }
- return numValues;
- }
-
- @Override
- protected void merge(MergeState state) throws IOException {
- if (datOut == null) {
- initDataOut();
- }
- if (state.liveDocs == null && state.reader instanceof FloatsReader) {
- // no deletes - bulk copy
- final FloatsReader reader = (FloatsReader) state.reader;
- assert reader.precisionBytes == (int) precision;
- if (reader.maxDoc == 0)
- return;
- final int docBase = state.docBase;
- if (docBase - lastDocId > 1) {
- // fill with default values
- lastDocId += fillDefault(docBase - lastDocId - 1);
- }
- lastDocId += reader.transferTo(datOut);
- } else {
- super.merge(state);
- }
-
- }
-
- @Override
- public void files(Collection<String> files) throws IOException {
- files.add(IndexFileNames.segmentFileName(id, "", Writer.DATA_EXTENSION));
- }
- }
-
- // Writes 4 bytes (float) per value
- static final class Float4Writer extends FloatsWriter {
- private int[] values;
- protected Float4Writer(Directory dir, String id, Counter bytesUsed, IOContext context)
- throws IOException {
- super(dir, id, 4, bytesUsed, context);
- values = new int[1];
- bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT);
- }
-
- @Override
- public void add(final int docID, final double v)
- throws IOException {
- assert docID > lastDocId : "docID: " + docID
- + " must be greater than the last added doc id: " + lastDocId;
- if (docID >= values.length) {
- final long len = values.length;
- values = ArrayUtil.grow(values, 1 + docID);
- bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT
- * ((values.length) - len));
- }
- values[docID] = Float.floatToRawIntBits((float)v);
- lastDocId = docID;
- }
-
- @Override
- protected void mergeDoc(int docID) throws IOException {
- assert datOut != null;
- assert docID > lastDocId : "docID: " + docID
- + " must be greater than the last added doc id: " + lastDocId;
- if (docID - lastDocId > 1) {
- // fill with default values
- fillDefault(docID - lastDocId - 1);
- }
- assert datOut != null;
- datOut.writeInt(Float.floatToRawIntBits((float) floatsRef.get()));
- lastDocId = docID;
- }
-
- @Override
- public void finish(int docCount) throws IOException {
- boolean success = false;
- try {
- int numDefaultsToAppend = docCount - (lastDocId + 1);
- if (datOut == null) {
- initDataOut();
- for (int i = 0; i <= lastDocId; i++) {
- datOut.writeInt(values[i]);
- }
- }
- fillDefault(numDefaultsToAppend);
- success = true;
- } finally {
- bytesUsed.addAndGet(-(RamUsageEstimator.NUM_BYTES_INT
- * ((values.length))));
- values = null;
- if (success) {
- IOUtils.close(datOut);
- } else {
- IOUtils.closeWhileHandlingException(datOut);
- }
- }
- }
}
- // Writes 8 bytes (double) per value
- static final class Float8Writer extends FloatsWriter {
- private long[] values;
- protected Float8Writer(Directory dir, String id, Counter bytesUsed, IOContext context)
- throws IOException {
- super(dir, id, 8, bytesUsed, context);
- values = new long[1];
- bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_LONG);
- }
-
- @Override
- public void add(int docID, double v) throws IOException {
- assert docID > lastDocId : "docID: " + docID
- + " must be greater than the last added doc id: " + lastDocId;
- if (docID >= values.length) {
- final long len = values.length;
- values = ArrayUtil.grow(values, 1 + docID);
- bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_LONG
- * ((values.length) - len));
+
+ final static class FloatsReader extends FixedStraightBytesImpl.Reader {
+ final IndexDocValuesArray arrayTemplate;
+ FloatsReader(Directory dir, String id, int maxDoc, IOContext context)
+ throws IOException {
+ super(dir, id, maxDoc, context);
+ assert size == 4 || size == 8;
+ if (size == 4) {
+ arrayTemplate = new IndexDocValuesArray.FloatValues();
+ } else {
+ arrayTemplate = new IndexDocValuesArray.DoubleValues();
}
- values[docID] = Double.doubleToLongBits(v);
- lastDocId = docID;
}
@Override
- protected void mergeDoc(int docID) throws IOException {
- assert docID > lastDocId : "docID: " + docID
- + " must be greater than the last added doc id: " + lastDocId;
- if (docID - lastDocId > 1) {
- // fill with default values
- lastDocId += fillDefault(docID - lastDocId - 1);
- }
- assert datOut != null;
- datOut.writeLong(Double.doubleToRawLongBits((float) floatsRef.get()));
- lastDocId = docID;
- }
-
- @Override
- public void finish(int docCount) throws IOException {
- boolean success = false;
- try {
- int numDefaultsToAppend = docCount - (lastDocId + 1);
- if (datOut == null) {
- initDataOut();
- for (int i = 0; i <= lastDocId; i++) {
- datOut.writeLong(values[i]);
- }
- }
- fillDefault(numDefaultsToAppend);
- success = true;
- } finally {
- bytesUsed.addAndGet(-(RamUsageEstimator.NUM_BYTES_LONG
- * ((values.length))));
- values = null;
- if (success) {
- IOUtils.close(datOut);
- } else {
- IOUtils.closeWhileHandlingException(datOut);
- }
- }
- }
- }
-
- /**
- * Opens all necessary files, but does not read any data in until you call
- * {@link #load}.
- */
- static class FloatsReader extends IndexDocValues {
-
- private final IndexInput datIn;
- private final int precisionBytes;
- // TODO(simonw) is ByteBuffer the way to go here?
- private final int maxDoc;
-
- protected FloatsReader(Directory dir, String id, int maxDoc, IOContext context)
- throws IOException {
- datIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
- Writer.DATA_EXTENSION), context);
- CodecUtil.checkHeader(datIn, CODEC_NAME, VERSION_START, VERSION_START);
- precisionBytes = datIn.readByte();
- assert precisionBytes == 4 || precisionBytes == 8;
- this.maxDoc = maxDoc;
- }
-
- int transferTo(IndexOutput out) throws IOException {
- IndexInput indexInput = (IndexInput) datIn.clone();
+ public Source load() throws IOException {
+ final IndexInput indexInput = cloneData();
try {
- indexInput.seek(CodecUtil.headerLength(CODEC_NAME));
- // skip precision:
- indexInput.readByte();
- out.copyBytes(indexInput, precisionBytes * maxDoc);
+ return arrayTemplate.newFromInput(indexInput, maxDoc);
} finally {
indexInput.close();
}
- return maxDoc;
}
-
- /**
- * Loads the actual values. You may call this more than once, eg if you
- * already previously loaded but then discarded the Source.
- */
- @Override
- public Source load() throws IOException {
- /* we always read BIG_ENDIAN here since the writer uses
- * DataOutput#writeInt() / writeLong() we can simply read the ints / longs
- * back in using readInt / readLong */
- final IndexInput indexInput = (IndexInput) datIn.clone();
- indexInput.seek(CodecUtil.headerLength(CODEC_NAME));
- // skip precision:
- indexInput.readByte();
- if (precisionBytes == 4) {
- final float[] values = new float[(4 * maxDoc) >> 2];
- assert values.length == maxDoc;
- for (int i = 0; i < values.length; i++) {
- values[i] = Float.intBitsToFloat(indexInput.readInt());
- }
- return new Source4(values);
- } else {
- final double[] values = new double[(8 * maxDoc) >> 3];
- assert values.length == maxDoc;
- for (int i = 0; i < values.length; i++) {
- values[i] = Double.longBitsToDouble(indexInput.readLong());
- }
- return new Source8(values);
- }
- }
-
- private final class Source4 extends Source {
- private final float[] values;
-
- Source4(final float[] values ) throws IOException {
- this.values = values;
- }
-
- @Override
- public double getFloat(int docID) {
- return values[docID];
- }
-
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource)
- throws IOException {
- return new SourceEnum(attrSource, ValueType.FLOAT_32, this, maxDoc) {
- @Override
- public int advance(int target) throws IOException {
- if (target >= numDocs)
- return pos = NO_MORE_DOCS;
- floatsRef.floats[floatsRef.offset] = source.getFloat(target);
- return pos = target;
- }
- };
- }
-
- @Override
- public Object getArray() {
- return this.values;
- }
-
- @Override
- public boolean hasArray() {
- return true;
- }
-
- @Override
- public ValueType type() {
- return ValueType.FLOAT_32;
- }
- }
-
- private final class Source8 extends Source {
- private final double[] values;
-
- Source8(final double[] values) throws IOException {
- this.values = values;
- }
-
- @Override
- public double getFloat(int docID) {
- return values[docID];
- }
-
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource)
- throws IOException {
- return new SourceEnum(attrSource, type(), this, maxDoc) {
- @Override
- public int advance(int target) throws IOException {
- if (target >= numDocs)
- return pos = NO_MORE_DOCS;
- floatsRef.floats[floatsRef.offset] = source.getFloat(target);
- return pos = target;
- }
- };
- }
-
- @Override
- public ValueType type() {
- return ValueType.FLOAT_64;
- }
-
- @Override
- public Object getArray() {
- return this.values;
- }
-
- @Override
- public boolean hasArray() {
- return true;
- }
- }
-
- @Override
- public void close() throws IOException {
- super.close();
- datIn.close();
- }
-
- @Override
+
public ValuesEnum getEnum(AttributeSource source) throws IOException {
IndexInput indexInput = (IndexInput) datIn.clone();
- indexInput.seek(CodecUtil.headerLength(CODEC_NAME));
- // skip precision:
- indexInput.readByte();
- return precisionBytes == 4 ? new Floats4Enum(source, indexInput, maxDoc)
- : new Floats8EnumImpl(source, indexInput, maxDoc);
+ return arrayTemplate.getDirectEnum(source, indexInput, maxDoc);
}
@Override
public ValueType type() {
- return precisionBytes == 4 ? ValueType.FLOAT_32
- : ValueType.FLOAT_64;
+ return arrayTemplate.type();
}
}
- static final class Floats4Enum extends FloatsEnumImpl {
-
- Floats4Enum(AttributeSource source, IndexInput dataIn, int maxDoc)
- throws IOException {
- super(source, dataIn, 4, maxDoc, ValueType.FLOAT_32);
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (target >= maxDoc)
- return pos = NO_MORE_DOCS;
- dataIn.seek(fp + (target * precision));
- final int intBits = dataIn.readInt();
- floatsRef.floats[0] = Float.intBitsToFloat(intBits);
- floatsRef.offset = 0;
- return pos = target;
- }
-
- @Override
- public int docID() {
- return pos;
- }
-
- @Override
- public int nextDoc() throws IOException {
- if (pos >= maxDoc) {
- return pos = NO_MORE_DOCS;
- }
- return advance(pos + 1);
- }
- }
-
- private static final class Floats8EnumImpl extends FloatsEnumImpl {
-
- Floats8EnumImpl(AttributeSource source, IndexInput dataIn, int maxDoc)
- throws IOException {
- super(source, dataIn, 8, maxDoc, ValueType.FLOAT_64);
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (target >= maxDoc) {
- return pos = NO_MORE_DOCS;
- }
- dataIn.seek(fp + (target * precision));
- final long value = dataIn.readLong();
- floatsRef.floats[floatsRef.offset] = Double.longBitsToDouble(value);
- return pos = target;
- }
-
- @Override
- public int docID() {
- return pos;
- }
-
- @Override
- public int nextDoc() throws IOException {
- if (pos >= maxDoc) {
- return pos = NO_MORE_DOCS;
- }
- return advance(pos + 1);
- }
- }
-
- static abstract class FloatsEnumImpl extends ValuesEnum {
- protected final IndexInput dataIn;
- protected int pos = -1;
- protected final int precision;
- protected final int maxDoc;
- protected final long fp;
-
- FloatsEnumImpl(AttributeSource source, IndexInput dataIn, int precision,
- int maxDoc, ValueType type) throws IOException {
- super(source, precision == 4 ? ValueType.FLOAT_32
- : ValueType.FLOAT_64);
- this.dataIn = dataIn;
- this.precision = precision;
- this.maxDoc = maxDoc;
- fp = dataIn.getFilePointer();
- floatsRef.offset = 0;
- }
-
- @Override
- public void close() throws IOException {
- dataIn.close();
- }
- }
}
\ No newline at end of file