You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/06/04 20:26:05 UTC
svn commit: r1600412 [2/3] - in /lucene/dev/trunk: lucene/
lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/
lucene/codecs/src/java/org/apache/lucene/codecs/memory/
lucene/core/src/java/org/apache/lucene/codecs/
lucene/core/src/java/org/apache/lu...
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java?rev=1600412&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java Wed Jun 4 18:26:04 2014
@@ -0,0 +1,908 @@
+package org.apache.lucene.codecs.lucene49;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.BINARY_FIXED_UNCOMPRESSED;
+import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.BINARY_PREFIX_COMPRESSED;
+import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.BINARY_VARIABLE_UNCOMPRESSED;
+import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.DELTA_COMPRESSED;
+import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.GCD_COMPRESSED;
+import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.MONOTONIC_COMPRESSED;
+import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.SORTED_SET_SINGLE_VALUED_SORTED;
+import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.SORTED_SET_WITH_ADDRESSES;
+import static org.apache.lucene.codecs.lucene49.Lucene49DocValuesConsumer.TABLE_COMPRESSED;
+
+import java.io.Closeable; // javadocs
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.RandomAccessOrds;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.TermsEnum.SeekStatus;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LongValues;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.packed.DirectReader;
+import org.apache.lucene.util.packed.MonotonicBlockPackedReader;
+
+/** reader for {@link Lucene49DocValuesFormat} */
+public class Lucene49DocValuesProducer extends DocValuesProducer implements Closeable {
+ private final Map<Integer,NumericEntry> numerics;
+ private final Map<Integer,BinaryEntry> binaries;
+ private final Map<Integer,SortedSetEntry> sortedSets;
+ private final Map<Integer,NumericEntry> ords;
+ private final Map<Integer,NumericEntry> ordIndexes;
+ private final AtomicLong ramBytesUsed;
+ private final IndexInput data;
+ private final int maxDoc;
+ private final int version;
+
+ // memory-resident structures
+ private final Map<Integer,MonotonicBlockPackedReader> addressInstances = new HashMap<>();
+ private final Map<Integer,MonotonicBlockPackedReader> ordIndexInstances = new HashMap<>();
+
+ /** expert: instantiates a new reader */
+ protected Lucene49DocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
+ String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
+ // read in the entries from the metadata file.
+ ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
+ this.maxDoc = state.segmentInfo.getDocCount();
+ boolean success = false;
+ try {
+ version = CodecUtil.checkHeader(in, metaCodec,
+ Lucene49DocValuesFormat.VERSION_START,
+ Lucene49DocValuesFormat.VERSION_CURRENT);
+ numerics = new HashMap<>();
+ ords = new HashMap<>();
+ ordIndexes = new HashMap<>();
+ binaries = new HashMap<>();
+ sortedSets = new HashMap<>();
+ readFields(in, state.fieldInfos);
+
+ CodecUtil.checkFooter(in);
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(in);
+ } else {
+ IOUtils.closeWhileHandlingException(in);
+ }
+ }
+
+ String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
+ this.data = state.directory.openInput(dataName, state.context);
+ success = false;
+ try {
+ final int version2 = CodecUtil.checkHeader(data, dataCodec,
+ Lucene49DocValuesFormat.VERSION_START,
+ Lucene49DocValuesFormat.VERSION_CURRENT);
+ if (version != version2) {
+ throw new CorruptIndexException("Format versions mismatch");
+ }
+
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(this.data);
+ }
+ }
+
+ ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
+ }
+
+ private void readSortedField(int fieldNumber, IndexInput meta, FieldInfos infos) throws IOException {
+ // sorted = binary + numeric
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene49DocValuesFormat.BINARY) {
+ throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ BinaryEntry b = readBinaryEntry(meta);
+ binaries.put(fieldNumber, b);
+
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene49DocValuesFormat.NUMERIC) {
+ throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ NumericEntry n = readNumericEntry(meta);
+ ords.put(fieldNumber, n);
+ }
+
+ private void readSortedSetFieldWithAddresses(int fieldNumber, IndexInput meta, FieldInfos infos) throws IOException {
+ // sortedset = binary + numeric (addresses) + ordIndex
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene49DocValuesFormat.BINARY) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ BinaryEntry b = readBinaryEntry(meta);
+ binaries.put(fieldNumber, b);
+
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene49DocValuesFormat.NUMERIC) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ NumericEntry n1 = readNumericEntry(meta);
+ ords.put(fieldNumber, n1);
+
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene49DocValuesFormat.NUMERIC) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ NumericEntry n2 = readNumericEntry(meta);
+ ordIndexes.put(fieldNumber, n2);
+ }
+
+ private void readFields(IndexInput meta, FieldInfos infos) throws IOException {
+ int fieldNumber = meta.readVInt();
+ while (fieldNumber != -1) {
+ if (infos.fieldInfo(fieldNumber) == null) {
+ // trickier to validate more: because we re-use for norms, because we use multiple entries
+ // for "composite" types like sortedset, etc.
+ throw new CorruptIndexException("Invalid field number: " + fieldNumber + " (resource=" + meta + ")");
+ }
+ byte type = meta.readByte();
+ if (type == Lucene49DocValuesFormat.NUMERIC) {
+ numerics.put(fieldNumber, readNumericEntry(meta));
+ } else if (type == Lucene49DocValuesFormat.BINARY) {
+ BinaryEntry b = readBinaryEntry(meta);
+ binaries.put(fieldNumber, b);
+ } else if (type == Lucene49DocValuesFormat.SORTED) {
+ readSortedField(fieldNumber, meta, infos);
+ } else if (type == Lucene49DocValuesFormat.SORTED_SET) {
+ SortedSetEntry ss = readSortedSetEntry(meta);
+ sortedSets.put(fieldNumber, ss);
+ if (ss.format == SORTED_SET_WITH_ADDRESSES) {
+ readSortedSetFieldWithAddresses(fieldNumber, meta, infos);
+ } else if (ss.format == SORTED_SET_SINGLE_VALUED_SORTED) {
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene49DocValuesFormat.SORTED) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ readSortedField(fieldNumber, meta, infos);
+ } else {
+ throw new AssertionError();
+ }
+ } else {
+ throw new CorruptIndexException("invalid type: " + type + ", resource=" + meta);
+ }
+ fieldNumber = meta.readVInt();
+ }
+ }
+
+ static NumericEntry readNumericEntry(IndexInput meta) throws IOException {
+ NumericEntry entry = new NumericEntry();
+ entry.format = meta.readVInt();
+ entry.missingOffset = meta.readLong();
+ entry.offset = meta.readLong();
+ entry.count = meta.readVLong();
+ switch(entry.format) {
+ case GCD_COMPRESSED:
+ entry.minValue = meta.readLong();
+ entry.gcd = meta.readLong();
+ entry.bitsPerValue = meta.readVInt();
+ break;
+ case TABLE_COMPRESSED:
+ final int uniqueValues = meta.readVInt();
+ if (uniqueValues > 256) {
+ throw new CorruptIndexException("TABLE_COMPRESSED cannot have more than 256 distinct values, input=" + meta);
+ }
+ entry.table = new long[uniqueValues];
+ for (int i = 0; i < uniqueValues; ++i) {
+ entry.table[i] = meta.readLong();
+ }
+ entry.bitsPerValue = meta.readVInt();
+ break;
+ case DELTA_COMPRESSED:
+ entry.minValue = meta.readLong();
+ entry.bitsPerValue = meta.readVInt();
+ break;
+ case MONOTONIC_COMPRESSED:
+ entry.packedIntsVersion = meta.readVInt();
+ entry.blockSize = meta.readVInt();
+ break;
+ default:
+ throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
+ }
+ entry.endOffset = meta.readLong();
+ return entry;
+ }
+
+ static BinaryEntry readBinaryEntry(IndexInput meta) throws IOException {
+ BinaryEntry entry = new BinaryEntry();
+ entry.format = meta.readVInt();
+ entry.missingOffset = meta.readLong();
+ entry.minLength = meta.readVInt();
+ entry.maxLength = meta.readVInt();
+ entry.count = meta.readVLong();
+ entry.offset = meta.readLong();
+ switch(entry.format) {
+ case BINARY_FIXED_UNCOMPRESSED:
+ break;
+ case BINARY_PREFIX_COMPRESSED:
+ entry.addressInterval = meta.readVInt();
+ entry.addressesOffset = meta.readLong();
+ entry.packedIntsVersion = meta.readVInt();
+ entry.blockSize = meta.readVInt();
+ break;
+ case BINARY_VARIABLE_UNCOMPRESSED:
+ entry.addressesOffset = meta.readLong();
+ entry.packedIntsVersion = meta.readVInt();
+ entry.blockSize = meta.readVInt();
+ break;
+ default:
+ throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
+ }
+ return entry;
+ }
+
+ SortedSetEntry readSortedSetEntry(IndexInput meta) throws IOException {
+ SortedSetEntry entry = new SortedSetEntry();
+ entry.format = meta.readVInt();
+ if (entry.format != SORTED_SET_SINGLE_VALUED_SORTED && entry.format != SORTED_SET_WITH_ADDRESSES) {
+ throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
+ }
+ return entry;
+ }
+
+ @Override
+ public NumericDocValues getNumeric(FieldInfo field) throws IOException {
+ NumericEntry entry = numerics.get(field.number);
+ return getNumeric(entry);
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ return ramBytesUsed.get();
+ }
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ CodecUtil.checksumEntireFile(data);
+ }
+
+ LongValues getNumeric(NumericEntry entry) throws IOException {
+ RandomAccessInput slice = this.data.randomAccessSlice(entry.offset, entry.endOffset - entry.offset);
+
+ switch (entry.format) {
+ case DELTA_COMPRESSED:
+ final long delta = entry.minValue;
+ final LongValues values = DirectReader.getInstance(slice, entry.bitsPerValue);
+ return new LongValues() {
+ @Override
+ public long get(long id) {
+ return delta + values.get(id);
+ }
+ };
+ case GCD_COMPRESSED:
+ final long min = entry.minValue;
+ final long mult = entry.gcd;
+ final LongValues quotientReader = DirectReader.getInstance(slice, entry.bitsPerValue);
+ return new LongValues() {
+ @Override
+ public long get(long id) {
+ return min + mult * quotientReader.get(id);
+ }
+ };
+ case TABLE_COMPRESSED:
+ final long table[] = entry.table;
+ final LongValues ords = DirectReader.getInstance(slice, entry.bitsPerValue);
+ return new LongValues() {
+ @Override
+ public long get(long id) {
+ return table[(int) ords.get(id)];
+ }
+ };
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ @Override
+ public BinaryDocValues getBinary(FieldInfo field) throws IOException {
+ BinaryEntry bytes = binaries.get(field.number);
+ switch(bytes.format) {
+ case BINARY_FIXED_UNCOMPRESSED:
+ return getFixedBinary(field, bytes);
+ case BINARY_VARIABLE_UNCOMPRESSED:
+ return getVariableBinary(field, bytes);
+ case BINARY_PREFIX_COMPRESSED:
+ return getCompressedBinary(field, bytes);
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ private BinaryDocValues getFixedBinary(FieldInfo field, final BinaryEntry bytes) {
+ final IndexInput data = this.data.clone();
+
+ return new LongBinaryDocValues() {
+ @Override
+ public void get(long id, BytesRef result) {
+ long address = bytes.offset + id * bytes.maxLength;
+ try {
+ data.seek(address);
+ // NOTE: we could have one buffer, but various consumers (e.g. FieldComparatorSource)
+ // assume "they" own the bytes after calling this!
+ final byte[] buffer = new byte[bytes.maxLength];
+ data.readBytes(buffer, 0, buffer.length);
+ result.bytes = buffer;
+ result.offset = 0;
+ result.length = buffer.length;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+
+ /** returns an address instance for variable-length binary values.
+ * @lucene.internal */
+ protected MonotonicBlockPackedReader getAddressInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException {
+ final MonotonicBlockPackedReader addresses;
+ synchronized (addressInstances) {
+ MonotonicBlockPackedReader addrInstance = addressInstances.get(field.number);
+ if (addrInstance == null) {
+ data.seek(bytes.addressesOffset);
+ addrInstance = new MonotonicBlockPackedReader(data, bytes.packedIntsVersion, bytes.blockSize, bytes.count, false);
+ addressInstances.put(field.number, addrInstance);
+ ramBytesUsed.addAndGet(addrInstance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
+ }
+ addresses = addrInstance;
+ }
+ return addresses;
+ }
+
+ private BinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException {
+ final IndexInput data = this.data.clone();
+
+ final MonotonicBlockPackedReader addresses = getAddressInstance(data, field, bytes);
+
+ return new LongBinaryDocValues() {
+ @Override
+ public void get(long id, BytesRef result) {
+ long startAddress = bytes.offset + (id == 0 ? 0 : addresses.get(id-1));
+ long endAddress = bytes.offset + addresses.get(id);
+ int length = (int) (endAddress - startAddress);
+ try {
+ data.seek(startAddress);
+ // NOTE: we could have one buffer, but various consumers (e.g. FieldComparatorSource)
+ // assume "they" own the bytes after calling this!
+ final byte[] buffer = new byte[length];
+ data.readBytes(buffer, 0, buffer.length);
+ result.bytes = buffer;
+ result.offset = 0;
+ result.length = length;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+
+ /** returns an address instance for prefix-compressed binary values.
+ * @lucene.internal */
+ protected MonotonicBlockPackedReader getIntervalInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException {
+ final MonotonicBlockPackedReader addresses;
+ final long interval = bytes.addressInterval;
+ synchronized (addressInstances) {
+ MonotonicBlockPackedReader addrInstance = addressInstances.get(field.number);
+ if (addrInstance == null) {
+ data.seek(bytes.addressesOffset);
+ final long size;
+ if (bytes.count % interval == 0) {
+ size = bytes.count / interval;
+ } else {
+ size = 1L + bytes.count / interval;
+ }
+ addrInstance = new MonotonicBlockPackedReader(data, bytes.packedIntsVersion, bytes.blockSize, size, false);
+ addressInstances.put(field.number, addrInstance);
+ ramBytesUsed.addAndGet(addrInstance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
+ }
+ addresses = addrInstance;
+ }
+ return addresses;
+ }
+
+
+ private BinaryDocValues getCompressedBinary(FieldInfo field, final BinaryEntry bytes) throws IOException {
+ final IndexInput data = this.data.clone();
+
+ final MonotonicBlockPackedReader addresses = getIntervalInstance(data, field, bytes);
+
+ return new CompressedBinaryDocValues(bytes, addresses, data);
+ }
+
+ @Override
+ public SortedDocValues getSorted(FieldInfo field) throws IOException {
+ final int valueCount = (int) binaries.get(field.number).count;
+ final BinaryDocValues binary = getBinary(field);
+ NumericEntry entry = ords.get(field.number);
+ final LongValues ordinals = getNumeric(entry);
+
+ return new SortedDocValues() {
+
+ @Override
+ public int getOrd(int docID) {
+ return (int) ordinals.get(docID);
+ }
+
+ @Override
+ public void lookupOrd(int ord, BytesRef result) {
+ binary.get(ord, result);
+ }
+
+ @Override
+ public int getValueCount() {
+ return valueCount;
+ }
+
+ @Override
+ public int lookupTerm(BytesRef key) {
+ if (binary instanceof CompressedBinaryDocValues) {
+ return (int) ((CompressedBinaryDocValues)binary).lookupTerm(key);
+ } else {
+ return super.lookupTerm(key);
+ }
+ }
+
+ @Override
+ public TermsEnum termsEnum() {
+ if (binary instanceof CompressedBinaryDocValues) {
+ return ((CompressedBinaryDocValues)binary).getTermsEnum();
+ } else {
+ return super.termsEnum();
+ }
+ }
+ };
+ }
+
+ /** returns an address instance for sortedset ordinal lists
+ * @lucene.internal */
+ protected MonotonicBlockPackedReader getOrdIndexInstance(IndexInput data, FieldInfo field, NumericEntry entry) throws IOException {
+ final MonotonicBlockPackedReader ordIndex;
+ synchronized (ordIndexInstances) {
+ MonotonicBlockPackedReader ordIndexInstance = ordIndexInstances.get(field.number);
+ if (ordIndexInstance == null) {
+ data.seek(entry.offset);
+ ordIndexInstance = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, false);
+ ordIndexInstances.put(field.number, ordIndexInstance);
+ ramBytesUsed.addAndGet(ordIndexInstance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
+ }
+ ordIndex = ordIndexInstance;
+ }
+ return ordIndex;
+ }
+
+ @Override
+ public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
+ SortedSetEntry ss = sortedSets.get(field.number);
+ if (ss.format == SORTED_SET_SINGLE_VALUED_SORTED) {
+ final SortedDocValues values = getSorted(field);
+ return DocValues.singleton(values);
+ } else if (ss.format != SORTED_SET_WITH_ADDRESSES) {
+ throw new AssertionError();
+ }
+
+ final IndexInput data = this.data.clone();
+ final long valueCount = binaries.get(field.number).count;
+ // we keep the byte[]s and list of ords on disk, these could be large
+ final LongBinaryDocValues binary = (LongBinaryDocValues) getBinary(field);
+ final LongValues ordinals = getNumeric(ords.get(field.number));
+ // but the addresses to the ord stream are in RAM
+ final MonotonicBlockPackedReader ordIndex = getOrdIndexInstance(data, field, ordIndexes.get(field.number));
+
+ return new RandomAccessOrds() {
+ long startOffset;
+ long offset;
+ long endOffset;
+
+ @Override
+ public long nextOrd() {
+ if (offset == endOffset) {
+ return NO_MORE_ORDS;
+ } else {
+ long ord = ordinals.get(offset);
+ offset++;
+ return ord;
+ }
+ }
+
+ @Override
+ public void setDocument(int docID) {
+ startOffset = offset = (docID == 0 ? 0 : ordIndex.get(docID-1));
+ endOffset = ordIndex.get(docID);
+ }
+
+ @Override
+ public void lookupOrd(long ord, BytesRef result) {
+ binary.get(ord, result);
+ }
+
+ @Override
+ public long getValueCount() {
+ return valueCount;
+ }
+
+ @Override
+ public long lookupTerm(BytesRef key) {
+ if (binary instanceof CompressedBinaryDocValues) {
+ return ((CompressedBinaryDocValues)binary).lookupTerm(key);
+ } else {
+ return super.lookupTerm(key);
+ }
+ }
+
+ @Override
+ public TermsEnum termsEnum() {
+ if (binary instanceof CompressedBinaryDocValues) {
+ return ((CompressedBinaryDocValues)binary).getTermsEnum();
+ } else {
+ return super.termsEnum();
+ }
+ }
+
+ @Override
+ public long ordAt(int index) {
+ return ordinals.get(startOffset + index);
+ }
+
+ @Override
+ public int cardinality() {
+ return (int) (endOffset - startOffset);
+ }
+ };
+ }
+
+ private Bits getMissingBits(final long offset) throws IOException {
+ if (offset == -1) {
+ return new Bits.MatchAllBits(maxDoc);
+ } else {
+ int length = (int) ((maxDoc + 7L) >>> 3);
+ final RandomAccessInput in = data.randomAccessSlice(offset, length);
+ return new Bits() {
+ @Override
+ public boolean get(int index) {
+ try {
+ return (in.readByte(index >> 3) & (1 << (index & 7))) != 0;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public int length() {
+ return maxDoc;
+ }
+ };
+ }
+ }
+
+ @Override
+ public Bits getDocsWithField(FieldInfo field) throws IOException {
+ switch(field.getDocValuesType()) {
+ case SORTED_SET:
+ return DocValues.docsWithValue(getSortedSet(field), maxDoc);
+ case SORTED:
+ return DocValues.docsWithValue(getSorted(field), maxDoc);
+ case BINARY:
+ BinaryEntry be = binaries.get(field.number);
+ return getMissingBits(be.missingOffset);
+ case NUMERIC:
+ NumericEntry ne = numerics.get(field.number);
+ return getMissingBits(ne.missingOffset);
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ data.close();
+ }
+
+ /** metadata entry for a numeric docvalues field */
+ protected static class NumericEntry {
+ private NumericEntry() {}
+ /** offset to the bitset representing docsWithField, or -1 if no documents have missing values */
+ long missingOffset;
+ /** offset to the actual numeric values */
+ public long offset;
+ /** end offset to the actual numeric values */
+ public long endOffset;
+ /** bits per value used to pack the numeric values */
+ public int bitsPerValue;
+
+ int format;
+ /** packed ints version used to encode these numerics */
+ public int packedIntsVersion;
+ /** count of values written */
+ public long count;
+ /** packed ints blocksize */
+ public int blockSize;
+
+ long minValue;
+ long gcd;
+ long table[];
+ }
+
+ /** metadata entry for a binary docvalues field */
+ protected static class BinaryEntry {
+ private BinaryEntry() {}
+ /** offset to the bitset representing docsWithField, or -1 if no documents have missing values */
+ long missingOffset;
+ /** offset to the actual binary values */
+ long offset;
+
+ int format;
+ /** count of values written */
+ public long count;
+ int minLength;
+ int maxLength;
+ /** offset to the addressing data that maps a value to its slice of the byte[] */
+ public long addressesOffset;
+ /** interval of shared prefix chunks (when using prefix-compressed binary) */
+ public long addressInterval;
+ /** packed ints version used to encode addressing information */
+ public int packedIntsVersion;
+ /** packed ints blocksize */
+ public int blockSize;
+ }
+
+ /** metadata entry for a sorted-set docvalues field */
+ protected static class SortedSetEntry {
+ private SortedSetEntry() {}
+ int format;
+ }
+
+ // internally we compose complex dv (sorted/sortedset) from other ones
+ static abstract class LongBinaryDocValues extends BinaryDocValues {
+ @Override
+ public final void get(int docID, BytesRef result) {
+ get((long)docID, result);
+ }
+
+ abstract void get(long id, BytesRef Result);
+ }
+
+ // in the compressed case, we add a few additional operations for
+ // more efficient reverse lookup and enumeration
+ static class CompressedBinaryDocValues extends LongBinaryDocValues {
+ final BinaryEntry bytes;
+ final long interval;
+ final long numValues;
+ final long numIndexValues;
+ final MonotonicBlockPackedReader addresses;
+ final IndexInput data;
+ final TermsEnum termsEnum;
+
+ public CompressedBinaryDocValues(BinaryEntry bytes, MonotonicBlockPackedReader addresses, IndexInput data) throws IOException {
+ this.bytes = bytes;
+ this.interval = bytes.addressInterval;
+ this.addresses = addresses;
+ this.data = data;
+ this.numValues = bytes.count;
+ this.numIndexValues = addresses.size();
+ this.termsEnum = getTermsEnum(data);
+ }
+
+ @Override
+ public void get(long id, BytesRef result) {
+ try {
+ termsEnum.seekExact(id);
+ BytesRef term = termsEnum.term();
+ result.bytes = term.bytes;
+ result.offset = term.offset;
+ result.length = term.length;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ long lookupTerm(BytesRef key) {
+ try {
+ SeekStatus status = termsEnum.seekCeil(key);
+ if (status == SeekStatus.END) {
+ return -numValues-1;
+ } else if (status == SeekStatus.FOUND) {
+ return termsEnum.ord();
+ } else {
+ return -termsEnum.ord()-1;
+ }
+ } catch (IOException bogus) {
+ throw new RuntimeException(bogus);
+ }
+ }
+
+ TermsEnum getTermsEnum() {
+ try {
+ return getTermsEnum(data.clone());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private TermsEnum getTermsEnum(final IndexInput input) throws IOException {
+ input.seek(bytes.offset);
+
+ return new TermsEnum() {
+ private long currentOrd = -1;
+ // TODO: maxLength is negative when all terms are merged away...
+ private final BytesRef termBuffer = new BytesRef(bytes.maxLength < 0 ? 0 : bytes.maxLength);
+ private final BytesRef term = new BytesRef(); // TODO: paranoia?
+
+ @Override
+ public BytesRef next() throws IOException {
+ if (doNext() == null) {
+ return null;
+ } else {
+ setTerm();
+ return term;
+ }
+ }
+
+ private BytesRef doNext() throws IOException {
+ if (++currentOrd >= numValues) {
+ return null;
+ } else {
+ int start = input.readVInt();
+ int suffix = input.readVInt();
+ input.readBytes(termBuffer.bytes, start, suffix);
+ termBuffer.length = start + suffix;
+ return termBuffer;
+ }
+ }
+
+ @Override
+ public SeekStatus seekCeil(BytesRef text) throws IOException {
+ // binary-search just the index values to find the block,
+ // then scan within the block
+ long low = 0;
+ long high = numIndexValues-1;
+
+ while (low <= high) {
+ long mid = (low + high) >>> 1;
+ doSeek(mid * interval);
+ int cmp = termBuffer.compareTo(text);
+
+ if (cmp < 0) {
+ low = mid + 1;
+ } else if (cmp > 0) {
+ high = mid - 1;
+ } else {
+ // we got lucky, found an indexed term
+ setTerm();
+ return SeekStatus.FOUND;
+ }
+ }
+
+ if (numIndexValues == 0) {
+ return SeekStatus.END;
+ }
+
+ // block before insertion point
+ long block = low-1;
+ doSeek(block < 0 ? -1 : block * interval);
+
+ while (doNext() != null) {
+ int cmp = termBuffer.compareTo(text);
+ if (cmp == 0) {
+ setTerm();
+ return SeekStatus.FOUND;
+ } else if (cmp > 0) {
+ setTerm();
+ return SeekStatus.NOT_FOUND;
+ }
+ }
+
+ return SeekStatus.END;
+ }
+
+ @Override
+ public void seekExact(long ord) throws IOException {
+ doSeek(ord);
+ setTerm();
+ }
+
+ private void doSeek(long ord) throws IOException {
+ long block = ord / interval;
+
+ if (ord >= currentOrd && block == currentOrd / interval) {
+ // seek within current block
+ } else {
+ // position before start of block
+ currentOrd = ord - ord % interval - 1;
+ input.seek(bytes.offset + addresses.get(block));
+ }
+
+ while (currentOrd < ord) {
+ doNext();
+ }
+ }
+
+ private void setTerm() {
+ // TODO: is there a cleaner way
+ term.bytes = new byte[termBuffer.length];
+ term.offset = 0;
+ term.copyBytes(termBuffer);
+ }
+
+ @Override
+ public BytesRef term() throws IOException {
+ return term;
+ }
+
+ @Override
+ public long ord() throws IOException {
+ return currentOrd;
+ }
+
+ @Override
+ public int docFreq() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long totalTermFreq() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ }
+}
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/package.html?rev=1600412&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/package.html (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/package.html Wed Jun 4 18:26:04 2014
@@ -0,0 +1,401 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+Lucene 4.9 file format.
+
+<h1>Apache Lucene - Index File Formats</h1>
+<div>
+<ul>
+<li><a href="#Introduction">Introduction</a></li>
+<li><a href="#Definitions">Definitions</a>
+ <ul>
+ <li><a href="#Inverted_Indexing">Inverted Indexing</a></li>
+ <li><a href="#Types_of_Fields">Types of Fields</a></li>
+ <li><a href="#Segments">Segments</a></li>
+ <li><a href="#Document_Numbers">Document Numbers</a></li>
+ </ul>
+</li>
+<li><a href="#Overview">Index Structure Overview</a></li>
+<li><a href="#File_Naming">File Naming</a></li>
+<li><a href="#file-names">Summary of File Extensions</a></li>
+ <ul>
+ <li><a href="#Lock_File">Lock File</a></li>
+ <li><a href="#History">History</a></li>
+ <li><a href="#Limitations">Limitations</a></li>
+ </ul>
+</ul>
+</div>
+<a name="Introduction"></a>
+<h2>Introduction</h2>
+<div>
+<p>This document defines the index file formats used in this version of Lucene.
+If you are using a different version of Lucene, please consult the copy of
+<code>docs/</code> that was distributed with
+the version you are using.</p>
+<p>Apache Lucene is written in Java, but several efforts are underway to write
+<a href="http://wiki.apache.org/lucene-java/LuceneImplementations">versions of
+Lucene in other programming languages</a>. If these versions are to remain
+compatible with Apache Lucene, then a language-independent definition of the
+Lucene index format is required. This document thus attempts to provide a
+complete and independent definition of the Apache Lucene file formats.</p>
+<p>As Lucene evolves, this document should evolve. Versions of Lucene in
+different programming languages should endeavor to agree on file formats, and
+generate new versions of this document.</p>
+</div>
+<a name="Definitions" id="Definitions"></a>
+<h2>Definitions</h2>
+<div>
+<p>The fundamental concepts in Lucene are index, document, field and term.</p>
+<p>An index contains a sequence of documents.</p>
+<ul>
+<li>A document is a sequence of fields.</li>
+<li>A field is a named sequence of terms.</li>
+<li>A term is a sequence of bytes.</li>
+</ul>
+<p>The same sequence of bytes in two different fields is considered a different
+term. Thus terms are represented as a pair: the string naming the field, and the
+bytes within the field.</p>
+<a name="Inverted_Indexing"></a>
+<h3>Inverted Indexing</h3>
+<p>The index stores statistics about terms in order to make term-based search
+more efficient. Lucene's index falls into the family of indexes known as an
+<i>inverted index.</i> This is because it can list, for a term, the documents
+that contain it. This is the inverse of the natural relationship, in which
+documents list terms.</p>
+<a name="Types_of_Fields"></a>
+<h3>Types of Fields</h3>
+<p>In Lucene, fields may be <i>stored</i>, in which case their text is stored
+in the index literally, in a non-inverted manner. Fields that are inverted are
+called <i>indexed</i>. A field may be both stored and indexed.</p>
+<p>The text of a field may be <i>tokenized</i> into terms to be indexed, or the
+text of a field may be used literally as a term to be indexed. Most fields are
+tokenized, but sometimes it is useful for certain identifier fields to be
+indexed literally.</p>
+<p>See the {@link org.apache.lucene.document.Field Field}
+java docs for more information on Fields.</p>
+<a name="Segments" id="Segments"></a>
+<h3>Segments</h3>
+<p>Lucene indexes may be composed of multiple sub-indexes, or <i>segments</i>.
+Each segment is a fully independent index, which could be searched separately.
+Indexes evolve by:</p>
+<ol>
+<li>Creating new segments for newly added documents.</li>
+<li>Merging existing segments.</li>
+</ol>
+<p>Searches may involve multiple segments and/or multiple indexes, each index
+potentially composed of a set of segments.</p>
+<a name="Document_Numbers"></a>
+<h3>Document Numbers</h3>
+<p>Internally, Lucene refers to documents by an integer <i>document number</i>.
+The first document added to an index is numbered zero, and each subsequent
+document added gets a number one greater than the previous.</p>
+<p>Note that a document's number may change, so caution should be taken when
+storing these numbers outside of Lucene. In particular, numbers may change in
+the following situations:</p>
+<ul>
+<li>
+<p>The numbers stored in each segment are unique only within the segment, and
+must be converted before they can be used in a larger context. The standard
+technique is to allocate each segment a range of values, based on the range of
+numbers used in that segment. To convert a document number from a segment to an
+external value, the segment's <i>base</i> document number is added. To convert
+an external value back to a segment-specific value, the segment is identified
+by the range that the external value is in, and the segment's base value is
+subtracted. For example two five document segments might be combined, so that
+the first segment has a base value of zero, and the second of five. Document
+three from the second segment would have an external value of eight.</p>
+</li>
+<li>
+<p>When documents are deleted, gaps are created in the numbering. These are
+eventually removed as the index evolves through merging. Deleted documents are
+dropped when segments are merged. A freshly-merged segment thus has no gaps in
+its numbering.</p>
+</li>
+</ul>
+</div>
+<a name="Overview" id="Overview"></a>
+<h2>Index Structure Overview</h2>
+<div>
+<p>Each segment index maintains the following:</p>
+<ul>
+<li>
+{@link org.apache.lucene.codecs.lucene46.Lucene46SegmentInfoFormat Segment info}.
+ This contains metadata about a segment, such as the number of documents,
+ what files it uses,
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene46.Lucene46FieldInfosFormat Field names}.
+ This contains the set of field names used in the index.
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Stored Field values}.
+This contains, for each document, a list of attribute-value pairs, where the attributes
+are field names. These are used to store auxiliary information about the document, such as
+its title, url, or an identifier to access a database. The set of stored fields are what is
+returned for each hit when searching. This is keyed by document number.
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term dictionary}.
+A dictionary containing all of the terms used in all of the
+indexed fields of all of the documents. The dictionary also contains the number
+of documents which contain the term, and pointers to the term's frequency and
+proximity data.
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Frequency data}.
+For each term in the dictionary, the numbers of all the
+documents that contain that term, and the frequency of the term in that
+document, unless frequencies are omitted (IndexOptions.DOCS_ONLY)
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Proximity data}.
+For each term in the dictionary, the positions that the
+term occurs in each document. Note that this will not exist if all fields in
+all documents omit position data.
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene42.Lucene42NormsFormat Normalization factors}.
+For each field in each document, a value is stored
+that is multiplied into the score for hits on that field.
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat Term Vectors}.
+For each field in each document, the term vector (sometimes
+called document vector) may be stored. A term vector consists of term text and
+term frequency. To add Term Vectors to your index see the
+{@link org.apache.lucene.document.Field Field} constructors
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene49.Lucene49DocValuesFormat Per-document values}.
+Like stored values, these are also keyed by document
+number, but are generally intended to be loaded into main memory for fast
+access. Whereas stored values are generally intended for summary results from
+searches, per-document values are useful for things like scoring factors.
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat Deleted documents}.
+An optional file indicating which documents are deleted.
+</li>
+</ul>
+<p>Details on each of these are provided in their linked pages.</p>
+</div>
+<a name="File_Naming"></a>
+<h2>File Naming</h2>
+<div>
+<p>All files belonging to a segment have the same name with varying extensions.
+The extensions correspond to the different file formats described below. When
+using the Compound File format (default in 1.4 and greater) these files (except
+for the Segment info file, the Lock file, and Deleted documents file) are collapsed
+into a single .cfs file (see below for details)</p>
+<p>Typically, all segments in an index are stored in a single directory,
+although this is not required.</p>
+<p>As of version 2.1 (lock-less commits), file names are never re-used (there
+is one exception, "segments.gen", see below). That is, when any file is saved
+to the Directory it is given a never before used filename. This is achieved
+using a simple generations approach. For example, the first segments file is
+segments_1, then segments_2, etc. The generation is a sequential long integer
+represented in alpha-numeric (base 36) form.</p>
+</div>
+<a name="file-names" id="file-names"></a>
+<h2>Summary of File Extensions</h2>
+<div>
+<p>The following table summarizes the names and extensions of the files in
+Lucene:</p>
+<table cellspacing="1" cellpadding="4">
+<tr>
+<th>Name</th>
+<th>Extension</th>
+<th>Brief Description</th>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.index.SegmentInfos Segments File}</td>
+<td>segments.gen, segments_N</td>
+<td>Stores information about a commit point</td>
+</tr>
+<tr>
+<td><a href="#Lock_File">Lock File</a></td>
+<td>write.lock</td>
+<td>The Write lock prevents multiple IndexWriters from writing to the same
+file.</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat Segment Info}</td>
+<td>.si</td>
+<td>Stores metadata about a segment</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.store.CompoundFileDirectory Compound File}</td>
+<td>.cfs, .cfe</td>
+<td>An optional "virtual" file consisting of all the other index files for
+systems that frequently run out of file handles.</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene46.Lucene46FieldInfosFormat Fields}</td>
+<td>.fnm</td>
+<td>Stores information about the fields</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Field Index}</td>
+<td>.fdx</td>
+<td>Contains pointers to field data</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Field Data}</td>
+<td>.fdt</td>
+<td>The stored fields for documents</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Dictionary}</td>
+<td>.tim</td>
+<td>The term dictionary, stores term info</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Index}</td>
+<td>.tip</td>
+<td>The index into the Term Dictionary</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Frequencies}</td>
+<td>.doc</td>
+<td>Contains the list of docs which contain each term along with frequency</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Positions}</td>
+<td>.pos</td>
+<td>Stores position information about where a term occurs in the index</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Payloads}</td>
+<td>.pay</td>
+<td>Stores additional per-position metadata information such as character offsets and user payloads</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene42.Lucene42NormsFormat Norms}</td>
+<td>.nvd, .nvm</td>
+<td>Encodes length and boost factors for docs and fields</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene49.Lucene49DocValuesFormat Per-Document Values}</td>
+<td>.dvd, .dvm</td>
+<td>Encodes additional scoring factors or other per-document information.</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat Term Vector Index}</td>
+<td>.tvx</td>
+<td>Stores offset into the document data file</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat Term Vector Documents}</td>
+<td>.tvd</td>
+<td>Contains information about each document that has term vectors</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat Term Vector Fields}</td>
+<td>.tvf</td>
+<td>The field level info about term vectors</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat Deleted Documents}</td>
+<td>.del</td>
+<td>Info about what files are deleted</td>
+</tr>
+</table>
+</div>
+<a name="Lock_File" id="Lock_File"></a>
+<h2>Lock File</h2>
+The write lock, which is stored in the index directory by default, is named
+"write.lock". If the lock directory is different from the index directory then
+the write lock will be named "XXXX-write.lock" where XXXX is a unique prefix
+derived from the full path to the index directory. When this file is present, a
+writer is currently modifying the index (adding or removing documents). This
+lock file ensures that only one writer is modifying the index at a time.</p>
+<a name="History"></a>
+<h2>History</h2>
+<p>Compatibility notes are provided in this document, describing how file
+formats have changed from prior versions:</p>
+<ul>
+<li>In version 2.1, the file format was changed to allow lock-less commits (ie,
+no more commit lock). The change is fully backwards compatible: you can open a
+pre-2.1 index for searching or adding/deleting of docs. When the new segments
+file is saved (committed), it will be written in the new file format (meaning
+no specific "upgrade" process is needed). But note that once a commit has
+occurred, pre-2.1 Lucene will not be able to read the index.</li>
+<li>In version 2.3, the file format was changed to allow segments to share a
+single set of doc store (vectors & stored fields) files. This allows for
+faster indexing in certain cases. The change is fully backwards compatible (in
+the same way as the lock-less commits change in 2.1).</li>
+<li>In version 2.4, Strings are now written as true UTF-8 byte sequence, not
+Java's modified UTF-8. See <a href="http://issues.apache.org/jira/browse/LUCENE-510">
+LUCENE-510</a> for details.</li>
+<li>In version 2.9, an optional opaque Map<String,String> CommitUserData
+may be passed to IndexWriter's commit methods (and later retrieved), which is
+recorded in the segments_N file. See <a href="http://issues.apache.org/jira/browse/LUCENE-1382">
+LUCENE-1382</a> for details. Also,
+diagnostics were added to each segment written recording details about why it
+was written (due to flush, merge; which OS/JRE was used; etc.). See issue
+<a href="http://issues.apache.org/jira/browse/LUCENE-1654">LUCENE-1654</a> for details.</li>
+<li>In version 3.0, compressed fields are no longer written to the index (they
+can still be read, but on merge the new segment will write them, uncompressed).
+See issue <a href="http://issues.apache.org/jira/browse/LUCENE-1960">LUCENE-1960</a>
+for details.</li>
+<li>In version 3.1, segments records the code version that created them. See
+<a href="http://issues.apache.org/jira/browse/LUCENE-2720">LUCENE-2720</a> for details.
+Additionally segments track explicitly whether or not they have term vectors.
+See <a href="http://issues.apache.org/jira/browse/LUCENE-2811">LUCENE-2811</a>
+for details.</li>
+<li>In version 3.2, numeric fields are written as natively to stored fields
+file, previously they were stored in text format only.</li>
+<li>In version 3.4, fields can omit position data while still indexing term
+frequencies.</li>
+<li>In version 4.0, the format of the inverted index became extensible via
+the {@link org.apache.lucene.codecs.Codec Codec} api. Fast per-document storage
+({@code DocValues}) was introduced. Normalization factors need no longer be a
+single byte, they can be any {@link org.apache.lucene.index.NumericDocValues NumericDocValues}.
+Terms need not be unicode strings, they can be any byte sequence. Term offsets
+can optionally be indexed into the postings lists. Payloads can be stored in the
+term vectors.</li>
+<li>In version 4.1, the format of the postings list changed to use either
+of FOR compression or variable-byte encoding, depending upon the frequency
+of the term. Terms appearing only once were changed to inline directly into
+the term dictionary. Stored fields are compressed by default. </li>
+<li>In version 4.2, term vectors are compressed by default. DocValues has
+a new multi-valued type (SortedSet), that can be used for faceting/grouping/joining
+on multi-valued fields.</li>
+<li>In version 4.5, DocValues were extended to explicitly represent missing values.</li>
+<li>In version 4.6, FieldInfos were extended to support per-field DocValues generation, to
+allow updating NumericDocValues fields.</li>
+<li>In version 4.8, checksum footers were added to the end of each index file
+for improved data integrity. Specifically, the last 8 bytes of every index file
+contain the zlib-crc32 checksum of the file.</li>
+</ul>
+<a name="Limitations" id="Limitations"></a>
+<h2>Limitations</h2>
+<div>
+<p>Lucene uses a Java <code>int</code> to refer to
+document numbers, and the index file format uses an <code>Int32</code>
+on-disk to store document numbers. This is a limitation
+of both the index file format and the current implementation. Eventually these
+should be replaced with either <code>UInt64</code> values, or
+better yet, {@link org.apache.lucene.store.DataOutput#writeVInt VInt} values which have no limit.</p>
+</div>
+</body>
+</html>
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/package.html?rev=1600412&r1=1600411&r2=1600412&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/package.html (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/package.html Wed Jun 4 18:26:04 2014
@@ -61,13 +61,13 @@ name of your codec.
If you just want to customise the {@link org.apache.lucene.codecs.PostingsFormat}, or use different postings
formats for different fields, then you can register your custom postings format in the same way (in
META-INF/services/org.apache.lucene.codecs.PostingsFormat), and then extend the default
- {@link org.apache.lucene.codecs.lucene46.Lucene46Codec} and override
- {@link org.apache.lucene.codecs.lucene46.Lucene46Codec#getPostingsFormatForField(String)} to return your custom
+ {@link org.apache.lucene.codecs.lucene49.Lucene49Codec} and override
+ {@link org.apache.lucene.codecs.lucene49.Lucene49Codec#getPostingsFormatForField(String)} to return your custom
postings format.
</p>
<p>
Similarly, if you just want to customise the {@link org.apache.lucene.codecs.DocValuesFormat} per-field, have
- a look at {@link org.apache.lucene.codecs.lucene46.Lucene46Codec#getDocValuesFormatForField(String)}.
+ a look at {@link org.apache.lucene.codecs.lucene49.Lucene49Codec#getDocValuesFormatForField(String)}.
</p>
</body>
</html>
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java?rev=1600412&r1=1600411&r2=1600412&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java Wed Jun 4 18:26:04 2014
@@ -36,7 +36,7 @@ import org.apache.lucene.util.WeakIdenti
* For efficiency, this class requires that the buffers
* are a power-of-two (<code>chunkSizePower</code>).
*/
-abstract class ByteBufferIndexInput extends IndexInput {
+abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessInput {
protected final BufferCleaner cleaner;
protected final long length;
protected final long chunkSizeMask;
@@ -178,6 +178,76 @@ abstract class ByteBufferIndexInput exte
throw new AlreadyClosedException("Already closed: " + this);
}
}
+
+ @Override
+ public byte readByte(long pos) throws IOException {
+ try {
+ final int bi = (int) (pos >> chunkSizePower);
+ return buffers[bi].get((int) (pos & chunkSizeMask));
+ } catch (IndexOutOfBoundsException ioobe) {
+ throw new EOFException("seek past EOF: " + this);
+ } catch (NullPointerException npe) {
+ throw new AlreadyClosedException("Already closed: " + this);
+ }
+ }
+
+ // used only by random access methods to handle reads across boundaries
+ private void setPos(long pos, int bi) throws IOException {
+ try {
+ final ByteBuffer b = buffers[bi];
+ b.position((int) (pos & chunkSizeMask));
+ this.curBufIndex = bi;
+ this.curBuf = b;
+ } catch (ArrayIndexOutOfBoundsException aioobe) {
+ throw new EOFException("seek past EOF: " + this);
+ } catch (IllegalArgumentException iae) {
+ throw new EOFException("seek past EOF: " + this);
+ } catch (NullPointerException npe) {
+ throw new AlreadyClosedException("Already closed: " + this);
+ }
+ }
+
+ @Override
+ public short readShort(long pos) throws IOException {
+ final int bi = (int) (pos >> chunkSizePower);
+ try {
+ return buffers[bi].getShort((int) (pos & chunkSizeMask));
+ } catch (IndexOutOfBoundsException ioobe) {
+ // either its a boundary, or read past EOF, fall back:
+ setPos(pos, bi);
+ return readShort();
+ } catch (NullPointerException npe) {
+ throw new AlreadyClosedException("Already closed: " + this);
+ }
+ }
+
+ @Override
+ public int readInt(long pos) throws IOException {
+ final int bi = (int) (pos >> chunkSizePower);
+ try {
+ return buffers[bi].getInt((int) (pos & chunkSizeMask));
+ } catch (IndexOutOfBoundsException ioobe) {
+ // either its a boundary, or read past EOF, fall back:
+ setPos(pos, bi);
+ return readInt();
+ } catch (NullPointerException npe) {
+ throw new AlreadyClosedException("Already closed: " + this);
+ }
+ }
+
+ @Override
+ public long readLong(long pos) throws IOException {
+ final int bi = (int) (pos >> chunkSizePower);
+ try {
+ return buffers[bi].getLong((int) (pos & chunkSizeMask));
+ } catch (IndexOutOfBoundsException ioobe) {
+ // either its a boundary, or read past EOF, fall back:
+ setPos(pos, bi);
+ return readLong();
+ } catch (NullPointerException npe) {
+ throw new AlreadyClosedException("Already closed: " + this);
+ }
+ }
@Override
public final long length() {
@@ -208,6 +278,12 @@ abstract class ByteBufferIndexInput exte
return buildSlice(sliceDescription, offset, length);
}
+ @Override
+ public RandomAccessInput randomAccessSlice(long offset, long length) throws IOException {
+ // note: technically we could even avoid the clone...
+ return slice(null, offset, length);
+ }
+
/** Builds the actual sliced IndexInput (may apply extra offset in subclasses). **/
protected ByteBufferIndexInput buildSlice(String sliceDescription, long offset, long length) {
if (buffers == null) {
@@ -373,6 +449,66 @@ abstract class ByteBufferIndexInput exte
throw new AlreadyClosedException("Already closed: " + this);
}
}
+
+ @Override
+ public byte readByte(long pos) throws IOException {
+ try {
+ return curBuf.get((int) pos);
+ } catch (IllegalArgumentException e) {
+ if (pos < 0) {
+ throw new IllegalArgumentException("Seeking to negative position: " + this, e);
+ } else {
+ throw new EOFException("seek past EOF: " + this);
+ }
+ } catch (NullPointerException npe) {
+ throw new AlreadyClosedException("Already closed: " + this);
+ }
+ }
+
+ @Override
+ public short readShort(long pos) throws IOException {
+ try {
+ return curBuf.getShort((int) pos);
+ } catch (IllegalArgumentException e) {
+ if (pos < 0) {
+ throw new IllegalArgumentException("Seeking to negative position: " + this, e);
+ } else {
+ throw new EOFException("seek past EOF: " + this);
+ }
+ } catch (NullPointerException npe) {
+ throw new AlreadyClosedException("Already closed: " + this);
+ }
+ }
+
+ @Override
+ public int readInt(long pos) throws IOException {
+ try {
+ return curBuf.getInt((int) pos);
+ } catch (IllegalArgumentException e) {
+ if (pos < 0) {
+ throw new IllegalArgumentException("Seeking to negative position: " + this, e);
+ } else {
+ throw new EOFException("seek past EOF: " + this);
+ }
+ } catch (NullPointerException npe) {
+ throw new AlreadyClosedException("Already closed: " + this);
+ }
+ }
+
+ @Override
+ public long readLong(long pos) throws IOException {
+ try {
+ return curBuf.getLong((int) pos);
+ } catch (IllegalArgumentException e) {
+ if (pos < 0) {
+ throw new IllegalArgumentException("Seeking to negative position: " + this, e);
+ } else {
+ throw new EOFException("seek past EOF: " + this);
+ }
+ } catch (NullPointerException npe) {
+ throw new AlreadyClosedException("Already closed: " + this);
+ }
+ }
}
/** This class adds offset support to ByteBufferIndexInput, which is needed for slices. */
@@ -405,6 +541,26 @@ abstract class ByteBufferIndexInput exte
}
@Override
+ public byte readByte(long pos) throws IOException {
+ return super.readByte(pos + offset);
+ }
+
+ @Override
+ public short readShort(long pos) throws IOException {
+ return super.readShort(pos + offset);
+ }
+
+ @Override
+ public int readInt(long pos) throws IOException {
+ return super.readInt(pos + offset);
+ }
+
+ @Override
+ public long readLong(long pos) throws IOException {
+ return super.readLong(pos + offset);
+ }
+
+ @Override
protected ByteBufferIndexInput buildSlice(String sliceDescription, long ofs, long length) {
return super.buildSlice(sliceDescription, this.offset + ofs, length);
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/IndexInput.java?rev=1600412&r1=1600411&r2=1600412&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/IndexInput.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/IndexInput.java Wed Jun 4 18:26:04 2014
@@ -88,4 +88,39 @@ public abstract class IndexInput extends
* The slice is seeked to the beginning.
*/
public abstract IndexInput slice(String sliceDescription, long offset, long length) throws IOException;
+
+ /**
+ * Creates a random-access slice of this index input, with the given offset and length.
+ * <p>
+ * The default implementation calls {@link #slice}, and implements absolute reads as
+ * seek+read.
+ */
+ public RandomAccessInput randomAccessSlice(long offset, long length) throws IOException {
+ final IndexInput slice = slice("randomaccess", offset, length);
+ return new RandomAccessInput() {
+ @Override
+ public byte readByte(long pos) throws IOException {
+ slice.seek(pos);
+ return slice.readByte();
+ }
+
+ @Override
+ public short readShort(long pos) throws IOException {
+ slice.seek(pos);
+ return slice.readShort();
+ }
+
+ @Override
+ public int readInt(long pos) throws IOException {
+ slice.seek(pos);
+ return slice.readInt();
+ }
+
+ @Override
+ public long readLong(long pos) throws IOException {
+ slice.seek(pos);
+ return slice.readLong();
+ }
+ };
+ }
}
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RandomAccessInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RandomAccessInput.java?rev=1600412&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RandomAccessInput.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RandomAccessInput.java Wed Jun 4 18:26:04 2014
@@ -0,0 +1,49 @@
+package org.apache.lucene.store;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+/**
+ * Random Access Index API.
+ * Unlike {@link IndexInput}, this has no concept of file position, all reads
+ * are absolute. However, like IndexInput, it is only intended for use by a single thread.
+ */
+public interface RandomAccessInput {
+
+ /**
+ * Reads a byte at the given position in the file
+ * @see DataInput#readByte
+ */
+ public byte readByte(long pos) throws IOException;
+ /**
+ * Reads a short at the given position in the file
+ * @see DataInput#readShort
+ */
+ public short readShort(long pos) throws IOException;
+ /**
+ * Reads an integer at the given position in the file
+ * @see DataInput#readInt
+ */
+ public int readInt(long pos) throws IOException;
+ /**
+ * Reads a long at the given position in the file
+ * @see DataInput#readLong
+ */
+ public long readLong(long pos) throws IOException;
+}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java?rev=1600412&r1=1600411&r2=1600412&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java Wed Jun 4 18:26:04 2014
@@ -58,16 +58,13 @@ import org.apache.lucene.store.DataOutpu
* @lucene.internal
*/
public final class BlockPackedWriter extends AbstractBlockPackedWriter {
- final float acceptableOverheadRatio;
/**
* Sole constructor.
* @param blockSize the number of values of a single block, must be a power of 2
- * @param acceptableOverheadRatio an acceptable overhead ratio per value
*/
- public BlockPackedWriter(DataOutput out, int blockSize, float acceptableOverheadRatio) {
+ public BlockPackedWriter(DataOutput out, int blockSize) {
super(out, blockSize);
- this.acceptableOverheadRatio = acceptableOverheadRatio;
}
protected void flush() throws IOException {
@@ -80,7 +77,6 @@ public final class BlockPackedWriter ext
final long delta = max - min;
int bitsRequired = delta < 0 ? 64 : delta == 0L ? 0 : PackedInts.bitsRequired(delta);
- bitsRequired = PackedInts.fastestDirectBits(bitsRequired, acceptableOverheadRatio);
if (bitsRequired == 64) {
// no need to delta-encode
min = 0L;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/DirectPackedReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/DirectPackedReader.java?rev=1600412&r1=1600411&r2=1600412&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/DirectPackedReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/DirectPackedReader.java Wed Jun 4 18:26:04 2014
@@ -22,6 +22,7 @@ import org.apache.lucene.store.IndexInpu
import java.io.IOException;
/* Reads directly from disk on each get */
+// just for back compat, use DirectReader/DirectWriter for more efficient impl
class DirectPackedReader extends PackedInts.ReaderImpl {
final IndexInput in;
final long startPointer;
@@ -98,258 +99,4 @@ class DirectPackedReader extends PackedI
public long ramBytesUsed() {
return 0;
}
-
- static class DirectPackedReader1 extends DirectPackedReader {
- DirectPackedReader1(int valueCount, IndexInput in) {
- super(1, valueCount, in);
- }
-
- @Override
- public long get(int index) {
- try {
- in.seek(startPointer + (index >>> 3));
- int shift = 7 - (index & 7);
- return (in.readByte() >>> shift) & 0x1;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class DirectPackedReader2 extends DirectPackedReader {
- DirectPackedReader2(int valueCount, IndexInput in) {
- super(2, valueCount, in);
- }
-
- @Override
- public long get(int index) {
- try {
- in.seek(startPointer + (index >>> 2));
- int shift = (3 - (index & 3)) << 1;
- return (in.readByte() >>> shift) & 0x3;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class DirectPackedReader4 extends DirectPackedReader {
- DirectPackedReader4(int valueCount, IndexInput in) {
- super(4, valueCount, in);
- }
-
- @Override
- public long get(int index) {
- try {
- in.seek(startPointer + (index >>> 1));
- int shift = ((index + 1) & 1) << 2;
- return (in.readByte() >>> shift) & 0xF;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class DirectPackedReader8 extends DirectPackedReader {
- DirectPackedReader8(int valueCount, IndexInput in) {
- super(8, valueCount, in);
- }
-
- @Override
- public long get(int index) {
- try {
- in.seek(startPointer + index);
- return in.readByte() & 0xFF;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class DirectPackedReader12 extends DirectPackedReader {
- DirectPackedReader12(int valueCount, IndexInput in) {
- super(12, valueCount, in);
- }
-
- @Override
- public long get(int index) {
- try {
- long offset = (index * 12L) >>> 3;
- in.seek(startPointer + offset);
- int shift = ((index + 1) & 1) << 2;
- return (in.readShort() >>> shift) & 0xFFF;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class DirectPackedReader16 extends DirectPackedReader {
- DirectPackedReader16(int valueCount, IndexInput in) {
- super(16, valueCount, in);
- }
-
- @Override
- public long get(int index) {
- try {
- in.seek(startPointer + (index<<1));
- return in.readShort() & 0xFFFF;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class DirectPackedReader20 extends DirectPackedReader {
- DirectPackedReader20(int valueCount, IndexInput in) {
- super(20, valueCount, in);
- }
-
- @Override
- public long get(int index) {
- try {
- long offset = (index * 20L) >>> 3;
- in.seek(startPointer + offset);
- int v = in.readShort() << 8 | (in.readByte() & 0xFF);
- int shift = ((index + 1) & 1) << 2;
- return (v >>> shift) & 0xFFFFF;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class DirectPackedReader24 extends DirectPackedReader {
- DirectPackedReader24(int valueCount, IndexInput in) {
- super(24, valueCount, in);
- }
-
- @Override
- public long get(int index) {
- try {
- in.seek(startPointer + (index*3));
- return (in.readShort() & 0xFFFF) << 8 | (in.readByte() & 0xFF);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class DirectPackedReader28 extends DirectPackedReader {
- DirectPackedReader28(int valueCount, IndexInput in) {
- super(28, valueCount, in);
- }
-
- @Override
- public long get(int index) {
- try {
- long offset = (index * 28L) >>> 3;
- in.seek(startPointer + offset);
- int shift = ((index + 1) & 1) << 2;
- return (in.readInt() >>> shift) & 0xFFFFFFFL;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class DirectPackedReader32 extends DirectPackedReader {
- DirectPackedReader32(int valueCount, IndexInput in) {
- super(32, valueCount, in);
- }
-
- @Override
- public long get(int index) {
- try {
- in.seek(startPointer + (index<<2));
- return in.readInt() & 0xFFFFFFFFL;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class DirectPackedReader40 extends DirectPackedReader {
- DirectPackedReader40(int valueCount, IndexInput in) {
- super(40, valueCount, in);
- }
-
- @Override
- public long get(int index) {
- try {
- in.seek(startPointer + (index*5));
- return (in.readInt() & 0xFFFFFFFFL) << 8 | (in.readByte() & 0xFF);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class DirectPackedReader48 extends DirectPackedReader {
- DirectPackedReader48(int valueCount, IndexInput in) {
- super(48, valueCount, in);
- }
-
- @Override
- public long get(int index) {
- try {
- in.seek(startPointer + (index*6));
- return (in.readInt() & 0xFFFFFFFFL) << 16 | (in.readShort() & 0xFFFF);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class DirectPackedReader56 extends DirectPackedReader {
- DirectPackedReader56(int valueCount, IndexInput in) {
- super(56, valueCount, in);
- }
-
- @Override
- public long get(int index) {
- try {
- in.seek(startPointer + (index*7));
- return (in.readInt() & 0xFFFFFFFFL) << 24 | (in.readShort() & 0xFFFF) << 8 | (in.readByte() & 0xFF);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class DirectPackedReader64 extends DirectPackedReader {
- DirectPackedReader64(int valueCount, IndexInput in) {
- super(64, valueCount, in);
- }
-
- @Override
- public long get(int index) {
- try {
- in.seek(startPointer + (index<<3));
- return in.readLong();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static DirectPackedReader getInstance(int bitsPerValue, int valueCount, IndexInput in) {
- switch(bitsPerValue) {
- case 1: return new DirectPackedReader1(valueCount, in);
- case 2: return new DirectPackedReader2(valueCount, in);
- case 4: return new DirectPackedReader4(valueCount, in);
- case 8: return new DirectPackedReader8(valueCount, in);
- case 12: return new DirectPackedReader12(valueCount, in);
- case 16: return new DirectPackedReader16(valueCount, in);
- case 20: return new DirectPackedReader20(valueCount, in);
- case 24: return new DirectPackedReader24(valueCount, in);
- case 28: return new DirectPackedReader28(valueCount, in);
- case 32: return new DirectPackedReader32(valueCount, in);
- case 40: return new DirectPackedReader40(valueCount, in);
- case 48: return new DirectPackedReader48(valueCount, in);
- case 56: return new DirectPackedReader56(valueCount, in);
- case 64: return new DirectPackedReader64(valueCount, in);
- default: return new DirectPackedReader(bitsPerValue, valueCount, in);
- }
- }
}
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/DirectReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/DirectReader.java?rev=1600412&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/DirectReader.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/DirectReader.java Wed Jun 4 18:26:04 2014
@@ -0,0 +1,313 @@
+package org.apache.lucene.util.packed;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.LongValues;
+
+/**
+ * Retrieves an instance previously written by {@link DirectWriter}
+ * <p>
+ * Example usage:
+ * <pre class="prettyprint">
+ * int bitsPerValue = 100;
+ * IndexInput in = dir.openInput("packed", IOContext.DEFAULT);
+ * LongValues values = DirectReader.getInstance(in.randomAccessSlice(start, end), bitsPerValue);
+ * for (int i = 0; i < numValues; i++) {
+ * long value = values.get(i);
+ * }
+ * </pre>
+ * @see DirectWriter
+ */
+public class DirectReader {
+
+ /**
+ * Retrieves an instance from the specified slice written decoding
+ * {@code bitsPerValue} for each value
+ */
+ public static LongValues getInstance(RandomAccessInput slice, int bitsPerValue) {
+ switch (bitsPerValue) {
+ case 1: return new DirectPackedReader1(slice);
+ case 2: return new DirectPackedReader2(slice);
+ case 4: return new DirectPackedReader4(slice);
+ case 8: return new DirectPackedReader8(slice);
+ case 12: return new DirectPackedReader12(slice);
+ case 16: return new DirectPackedReader16(slice);
+ case 20: return new DirectPackedReader20(slice);
+ case 24: return new DirectPackedReader24(slice);
+ case 28: return new DirectPackedReader28(slice);
+ case 32: return new DirectPackedReader32(slice);
+ case 40: return new DirectPackedReader40(slice);
+ case 48: return new DirectPackedReader48(slice);
+ case 56: return new DirectPackedReader56(slice);
+ case 64: return new DirectPackedReader64(slice);
+ default: throw new IllegalArgumentException("unsupported bitsPerValue: " + bitsPerValue);
+ }
+ }
+
+ static final class DirectPackedReader1 extends LongValues {
+ final RandomAccessInput in;
+
+ DirectPackedReader1(RandomAccessInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public long get(long index) {
+ try {
+ int shift = 7 - (int) (index & 7);
+ return (in.readByte(index >>> 3) >>> shift) & 0x1;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static final class DirectPackedReader2 extends LongValues {
+ final RandomAccessInput in;
+
+ DirectPackedReader2(RandomAccessInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public long get(long index) {
+ try {
+ int shift = (3 - (int)(index & 3)) << 1;
+ return (in.readByte(index >>> 2) >>> shift) & 0x3;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static final class DirectPackedReader4 extends LongValues {
+ final RandomAccessInput in;
+
+ DirectPackedReader4(RandomAccessInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public long get(long index) {
+ try {
+ int shift = (int) ((index + 1) & 1) << 2;
+ return (in.readByte(index >>> 1) >>> shift) & 0xF;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static final class DirectPackedReader8 extends LongValues {
+ final RandomAccessInput in;
+
+ DirectPackedReader8(RandomAccessInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public long get(long index) {
+ try {
+ return in.readByte(index) & 0xFF;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static final class DirectPackedReader12 extends LongValues {
+ final RandomAccessInput in;
+
+ DirectPackedReader12(RandomAccessInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public long get(long index) {
+ try {
+ long offset = (index * 12) >>> 3;
+ int shift = (int) ((index + 1) & 1) << 2;
+ return (in.readShort(offset) >>> shift) & 0xFFF;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static final class DirectPackedReader16 extends LongValues {
+ final RandomAccessInput in;
+
+ DirectPackedReader16(RandomAccessInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public long get(long index) {
+ try {
+ return in.readShort(index << 1) & 0xFFFF;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static final class DirectPackedReader20 extends LongValues {
+ final RandomAccessInput in;
+
+ DirectPackedReader20(RandomAccessInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public long get(long index) {
+ try {
+ long offset = (index * 20) >>> 3;
+ // TODO: clean this up...
+ int v = in.readInt(offset) >>> 8;
+ int shift = (int) ((index + 1) & 1) << 2;
+ return (v >>> shift) & 0xFFFFF;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static final class DirectPackedReader24 extends LongValues {
+ final RandomAccessInput in;
+
+ DirectPackedReader24(RandomAccessInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public long get(long index) {
+ try {
+ return in.readInt(index * 3) >>> 8;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static final class DirectPackedReader28 extends LongValues {
+ final RandomAccessInput in;
+
+ DirectPackedReader28(RandomAccessInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public long get(long index) {
+ try {
+ long offset = (index * 28) >>> 3;
+ int shift = (int) ((index + 1) & 1) << 2;
+ return (in.readInt(offset) >>> shift) & 0xFFFFFFFL;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static final class DirectPackedReader32 extends LongValues {
+ final RandomAccessInput in;
+
+ DirectPackedReader32(RandomAccessInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public long get(long index) {
+ try {
+ return in.readInt(index << 2) & 0xFFFFFFFFL;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static final class DirectPackedReader40 extends LongValues {
+ final RandomAccessInput in;
+
+ DirectPackedReader40(RandomAccessInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public long get(long index) {
+ try {
+ return in.readLong(index * 5) >>> 24;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static final class DirectPackedReader48 extends LongValues {
+ final RandomAccessInput in;
+
+ DirectPackedReader48(RandomAccessInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public long get(long index) {
+ try {
+ return in.readLong(index * 6) >>> 16;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static final class DirectPackedReader56 extends LongValues {
+ final RandomAccessInput in;
+
+ DirectPackedReader56(RandomAccessInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public long get(long index) {
+ try {
+ return in.readLong(index * 7) >>> 8;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static final class DirectPackedReader64 extends LongValues {
+ final RandomAccessInput in;
+
+ DirectPackedReader64(RandomAccessInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public long get(long index) {
+ try {
+ return in.readLong(index << 3);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}