You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/08/14 17:30:45 UTC
svn commit: r1617975 [2/3] - in /lucene/dev/trunk: lucene/
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/
lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/
lucene/core/src/java/org/apache/lucene/codecs/ lucene/core/src/ja...
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/Lucene410DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/Lucene410DocValuesProducer.java?rev=1617975&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/Lucene410DocValuesProducer.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/Lucene410DocValuesProducer.java Thu Aug 14 15:30:44 2014
@@ -0,0 +1,1070 @@
+package org.apache.lucene.codecs.lucene410;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.BINARY_FIXED_UNCOMPRESSED;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.BINARY_PREFIX_COMPRESSED;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.BINARY_VARIABLE_UNCOMPRESSED;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.DELTA_COMPRESSED;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.GCD_COMPRESSED;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.MONOTONIC_COMPRESSED;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.SORTED_SINGLE_VALUED;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.SORTED_WITH_ADDRESSES;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.TABLE_COMPRESSED;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.INTERVAL_SHIFT;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.INTERVAL_COUNT;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.INTERVAL_MASK;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.REVERSE_INTERVAL_SHIFT;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.REVERSE_INTERVAL_MASK;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.BLOCK_INTERVAL_SHIFT;
+import static org.apache.lucene.codecs.lucene410.Lucene410DocValuesConsumer.BLOCK_INTERVAL_MASK;
+
+import java.io.Closeable; // javadocs
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.RandomAccessOrds;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LongValues;
+import org.apache.lucene.util.PagedBytes;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.packed.DirectReader;
+import org.apache.lucene.util.packed.MonotonicBlockPackedReader;
+
+/** reader for {@link Lucene410DocValuesFormat} */
+class Lucene410DocValuesProducer extends DocValuesProducer implements Closeable {
+ private final Map<Integer,NumericEntry> numerics;
+ private final Map<Integer,BinaryEntry> binaries;
+ private final Map<Integer,SortedSetEntry> sortedSets;
+ private final Map<Integer,SortedSetEntry> sortedNumerics;
+ private final Map<Integer,NumericEntry> ords;
+ private final Map<Integer,NumericEntry> ordIndexes;
+ private final AtomicLong ramBytesUsed;
+ private final IndexInput data;
+ private final int maxDoc;
+ private final int version;
+
+ // memory-resident structures
+ private final Map<Integer,MonotonicBlockPackedReader> addressInstances = new HashMap<>();
+ private final Map<Integer,MonotonicBlockPackedReader> ordIndexInstances = new HashMap<>();
+ private final Map<Integer,ReverseTermsIndex> reverseIndexInstances = new HashMap<>();
+
+ /** expert: instantiates a new reader */
+ Lucene410DocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
+ String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
+ // read in the entries from the metadata file.
+ ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
+ this.maxDoc = state.segmentInfo.getDocCount();
+ boolean success = false;
+ try {
+ version = CodecUtil.checkHeader(in, metaCodec,
+ Lucene410DocValuesFormat.VERSION_START,
+ Lucene410DocValuesFormat.VERSION_CURRENT);
+ numerics = new HashMap<>();
+ ords = new HashMap<>();
+ ordIndexes = new HashMap<>();
+ binaries = new HashMap<>();
+ sortedSets = new HashMap<>();
+ sortedNumerics = new HashMap<>();
+ readFields(in, state.fieldInfos);
+
+ CodecUtil.checkFooter(in);
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(in);
+ } else {
+ IOUtils.closeWhileHandlingException(in);
+ }
+ }
+
+ String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
+ this.data = state.directory.openInput(dataName, state.context);
+ success = false;
+ try {
+ final int version2 = CodecUtil.checkHeader(data, dataCodec,
+ Lucene410DocValuesFormat.VERSION_START,
+ Lucene410DocValuesFormat.VERSION_CURRENT);
+ if (version != version2) {
+ throw new CorruptIndexException("Format versions mismatch");
+ }
+
+ // NOTE: data file is too costly to verify checksum against all the bytes on open,
+ // but for now we at least verify proper structure of the checksum footer: which looks
+ // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
+ // such as file truncation.
+ CodecUtil.retrieveChecksum(data);
+
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(this.data);
+ }
+ }
+
+ ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
+ }
+
+ private void readSortedField(int fieldNumber, IndexInput meta, FieldInfos infos) throws IOException {
+ // sorted = binary + numeric
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene410DocValuesFormat.BINARY) {
+ throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ BinaryEntry b = readBinaryEntry(meta);
+ binaries.put(fieldNumber, b);
+
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene410DocValuesFormat.NUMERIC) {
+ throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ NumericEntry n = readNumericEntry(meta);
+ ords.put(fieldNumber, n);
+ }
+
+ private void readSortedSetFieldWithAddresses(int fieldNumber, IndexInput meta, FieldInfos infos) throws IOException {
+ // sortedset = binary + numeric (addresses) + ordIndex
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene410DocValuesFormat.BINARY) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ BinaryEntry b = readBinaryEntry(meta);
+ binaries.put(fieldNumber, b);
+
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene410DocValuesFormat.NUMERIC) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ NumericEntry n1 = readNumericEntry(meta);
+ ords.put(fieldNumber, n1);
+
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene410DocValuesFormat.NUMERIC) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ NumericEntry n2 = readNumericEntry(meta);
+ ordIndexes.put(fieldNumber, n2);
+ }
+
+ private void readFields(IndexInput meta, FieldInfos infos) throws IOException {
+ int fieldNumber = meta.readVInt();
+ while (fieldNumber != -1) {
+ if (infos.fieldInfo(fieldNumber) == null) {
+ // trickier to validate more: because we re-use for norms, because we use multiple entries
+ // for "composite" types like sortedset, etc.
+ throw new CorruptIndexException("Invalid field number: " + fieldNumber + " (resource=" + meta + ")");
+ }
+ byte type = meta.readByte();
+ if (type == Lucene410DocValuesFormat.NUMERIC) {
+ numerics.put(fieldNumber, readNumericEntry(meta));
+ } else if (type == Lucene410DocValuesFormat.BINARY) {
+ BinaryEntry b = readBinaryEntry(meta);
+ binaries.put(fieldNumber, b);
+ } else if (type == Lucene410DocValuesFormat.SORTED) {
+ readSortedField(fieldNumber, meta, infos);
+ } else if (type == Lucene410DocValuesFormat.SORTED_SET) {
+ SortedSetEntry ss = readSortedSetEntry(meta);
+ sortedSets.put(fieldNumber, ss);
+ if (ss.format == SORTED_WITH_ADDRESSES) {
+ readSortedSetFieldWithAddresses(fieldNumber, meta, infos);
+ } else if (ss.format == SORTED_SINGLE_VALUED) {
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene410DocValuesFormat.SORTED) {
+ throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ readSortedField(fieldNumber, meta, infos);
+ } else {
+ throw new AssertionError();
+ }
+ } else if (type == Lucene410DocValuesFormat.SORTED_NUMERIC) {
+ SortedSetEntry ss = readSortedSetEntry(meta);
+ sortedNumerics.put(fieldNumber, ss);
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sortednumeric entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene410DocValuesFormat.NUMERIC) {
+ throw new CorruptIndexException("sortednumeric entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ numerics.put(fieldNumber, readNumericEntry(meta));
+ if (ss.format == SORTED_WITH_ADDRESSES) {
+ if (meta.readVInt() != fieldNumber) {
+ throw new CorruptIndexException("sortednumeric entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ if (meta.readByte() != Lucene410DocValuesFormat.NUMERIC) {
+ throw new CorruptIndexException("sortednumeric entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ }
+ NumericEntry ordIndex = readNumericEntry(meta);
+ ordIndexes.put(fieldNumber, ordIndex);
+ } else if (ss.format != SORTED_SINGLE_VALUED) {
+ throw new AssertionError();
+ }
+ } else {
+ throw new CorruptIndexException("invalid type: " + type + ", resource=" + meta);
+ }
+ fieldNumber = meta.readVInt();
+ }
+ }
+
+ static NumericEntry readNumericEntry(IndexInput meta) throws IOException {
+ NumericEntry entry = new NumericEntry();
+ entry.format = meta.readVInt();
+ entry.missingOffset = meta.readLong();
+ entry.offset = meta.readLong();
+ entry.count = meta.readVLong();
+ switch(entry.format) {
+ case GCD_COMPRESSED:
+ entry.minValue = meta.readLong();
+ entry.gcd = meta.readLong();
+ entry.bitsPerValue = meta.readVInt();
+ break;
+ case TABLE_COMPRESSED:
+ final int uniqueValues = meta.readVInt();
+ if (uniqueValues > 256) {
+ throw new CorruptIndexException("TABLE_COMPRESSED cannot have more than 256 distinct values, input=" + meta);
+ }
+ entry.table = new long[uniqueValues];
+ for (int i = 0; i < uniqueValues; ++i) {
+ entry.table[i] = meta.readLong();
+ }
+ entry.bitsPerValue = meta.readVInt();
+ break;
+ case DELTA_COMPRESSED:
+ entry.minValue = meta.readLong();
+ entry.bitsPerValue = meta.readVInt();
+ break;
+ case MONOTONIC_COMPRESSED:
+ entry.packedIntsVersion = meta.readVInt();
+ entry.blockSize = meta.readVInt();
+ break;
+ default:
+ throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
+ }
+ entry.endOffset = meta.readLong();
+ return entry;
+ }
+
+ static BinaryEntry readBinaryEntry(IndexInput meta) throws IOException {
+ BinaryEntry entry = new BinaryEntry();
+ entry.format = meta.readVInt();
+ entry.missingOffset = meta.readLong();
+ entry.minLength = meta.readVInt();
+ entry.maxLength = meta.readVInt();
+ entry.count = meta.readVLong();
+ entry.offset = meta.readLong();
+ switch(entry.format) {
+ case BINARY_FIXED_UNCOMPRESSED:
+ break;
+ case BINARY_PREFIX_COMPRESSED:
+ entry.addressesOffset = meta.readLong();
+ entry.packedIntsVersion = meta.readVInt();
+ entry.blockSize = meta.readVInt();
+ entry.reverseIndexOffset = meta.readLong();
+ break;
+ case BINARY_VARIABLE_UNCOMPRESSED:
+ entry.addressesOffset = meta.readLong();
+ entry.packedIntsVersion = meta.readVInt();
+ entry.blockSize = meta.readVInt();
+ break;
+ default:
+ throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
+ }
+ return entry;
+ }
+
+ SortedSetEntry readSortedSetEntry(IndexInput meta) throws IOException {
+ SortedSetEntry entry = new SortedSetEntry();
+ entry.format = meta.readVInt();
+ if (entry.format != SORTED_SINGLE_VALUED && entry.format != SORTED_WITH_ADDRESSES) {
+ throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
+ }
+ return entry;
+ }
+
+ @Override
+ public NumericDocValues getNumeric(FieldInfo field) throws IOException {
+ NumericEntry entry = numerics.get(field.number);
+ return getNumeric(entry);
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ return ramBytesUsed.get();
+ }
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ CodecUtil.checksumEntireFile(data);
+ }
+
+ LongValues getNumeric(NumericEntry entry) throws IOException {
+ RandomAccessInput slice = this.data.randomAccessSlice(entry.offset, entry.endOffset - entry.offset);
+ switch (entry.format) {
+ case DELTA_COMPRESSED:
+ final long delta = entry.minValue;
+ final LongValues values = DirectReader.getInstance(slice, entry.bitsPerValue);
+ return new LongValues() {
+ @Override
+ public long get(long id) {
+ return delta + values.get(id);
+ }
+ };
+ case GCD_COMPRESSED:
+ final long min = entry.minValue;
+ final long mult = entry.gcd;
+ final LongValues quotientReader = DirectReader.getInstance(slice, entry.bitsPerValue);
+ return new LongValues() {
+ @Override
+ public long get(long id) {
+ return min + mult * quotientReader.get(id);
+ }
+ };
+ case TABLE_COMPRESSED:
+ final long table[] = entry.table;
+ final LongValues ords = DirectReader.getInstance(slice, entry.bitsPerValue);
+ return new LongValues() {
+ @Override
+ public long get(long id) {
+ return table[(int) ords.get(id)];
+ }
+ };
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ @Override
+ public BinaryDocValues getBinary(FieldInfo field) throws IOException {
+ BinaryEntry bytes = binaries.get(field.number);
+ switch(bytes.format) {
+ case BINARY_FIXED_UNCOMPRESSED:
+ return getFixedBinary(field, bytes);
+ case BINARY_VARIABLE_UNCOMPRESSED:
+ return getVariableBinary(field, bytes);
+ case BINARY_PREFIX_COMPRESSED:
+ return getCompressedBinary(field, bytes);
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ private BinaryDocValues getFixedBinary(FieldInfo field, final BinaryEntry bytes) throws IOException {
+ final IndexInput data = this.data.slice("fixed-binary", bytes.offset, bytes.count * bytes.maxLength);
+
+ final BytesRef term = new BytesRef(bytes.maxLength);
+ final byte[] buffer = term.bytes;
+ final int length = term.length = bytes.maxLength;
+
+ return new LongBinaryDocValues() {
+ @Override
+ public BytesRef get(long id) {
+ try {
+ data.seek(id * length);
+ data.readBytes(buffer, 0, buffer.length);
+ return term;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+
+ /** returns an address instance for variable-length binary values. */
+ private MonotonicBlockPackedReader getAddressInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException {
+ final MonotonicBlockPackedReader addresses;
+ synchronized (addressInstances) {
+ MonotonicBlockPackedReader addrInstance = addressInstances.get(field.number);
+ if (addrInstance == null) {
+ data.seek(bytes.addressesOffset);
+ addrInstance = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, bytes.count+1, false);
+ addressInstances.put(field.number, addrInstance);
+ ramBytesUsed.addAndGet(addrInstance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
+ }
+ addresses = addrInstance;
+ }
+ return addresses;
+ }
+
+ private BinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException {
+ final MonotonicBlockPackedReader addresses = getAddressInstance(data, field, bytes);
+
+ final IndexInput data = this.data.slice("var-binary", bytes.offset, bytes.addressesOffset - bytes.offset);
+ final BytesRef term = new BytesRef(Math.max(0, bytes.maxLength));
+ final byte buffer[] = term.bytes;
+
+ return new LongBinaryDocValues() {
+ @Override
+ public BytesRef get(long id) {
+ long startAddress = addresses.get(id);
+ long endAddress = addresses.get(id+1);
+ int length = (int) (endAddress - startAddress);
+ try {
+ data.seek(startAddress);
+ data.readBytes(buffer, 0, length);
+ term.length = length;
+ return term;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+
+ /** returns an address instance for prefix-compressed binary values. */
+ private MonotonicBlockPackedReader getIntervalInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException {
+ final MonotonicBlockPackedReader addresses;
+ synchronized (addressInstances) {
+ MonotonicBlockPackedReader addrInstance = addressInstances.get(field.number);
+ if (addrInstance == null) {
+ data.seek(bytes.addressesOffset);
+ final long size = (bytes.count + INTERVAL_MASK) >>> INTERVAL_SHIFT;
+ addrInstance = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, size, false);
+ addressInstances.put(field.number, addrInstance);
+ ramBytesUsed.addAndGet(addrInstance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
+ }
+ addresses = addrInstance;
+ }
+ return addresses;
+ }
+
+ /** returns a reverse lookup instance for prefix-compressed binary values. */
+ private ReverseTermsIndex getReverseIndexInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException {
+ final ReverseTermsIndex index;
+ synchronized (reverseIndexInstances) {
+ ReverseTermsIndex instance = reverseIndexInstances.get(field.number);
+ if (instance == null) {
+ instance = new ReverseTermsIndex();
+ data.seek(bytes.reverseIndexOffset);
+ long size = (bytes.count + REVERSE_INTERVAL_MASK) >>> REVERSE_INTERVAL_SHIFT;
+ instance.termAddresses = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, size, false);
+ long dataSize = data.readVLong();
+ PagedBytes pagedBytes = new PagedBytes(15);
+ pagedBytes.copy(data, dataSize);
+ instance.terms = pagedBytes.freeze(true);
+ reverseIndexInstances.put(field.number, instance);
+ ramBytesUsed.addAndGet(instance.termAddresses.ramBytesUsed() + instance.terms.ramBytesUsed());
+ }
+ index = instance;
+ }
+ return index;
+ }
+
+
+ private BinaryDocValues getCompressedBinary(FieldInfo field, final BinaryEntry bytes) throws IOException {
+ final MonotonicBlockPackedReader addresses = getIntervalInstance(data, field, bytes);
+ final ReverseTermsIndex index = getReverseIndexInstance(data, field, bytes);
+ assert addresses.size() > 0; // we don't have to handle empty case
+ IndexInput slice = data.slice("terms", bytes.offset, bytes.addressesOffset - bytes.offset);
+ return new CompressedBinaryDocValues(bytes, addresses, index, slice);
+ }
+
+ @Override
+ public SortedDocValues getSorted(FieldInfo field) throws IOException {
+ final int valueCount = (int) binaries.get(field.number).count;
+ final BinaryDocValues binary = getBinary(field);
+ NumericEntry entry = ords.get(field.number);
+ final LongValues ordinals = getNumeric(entry);
+ return new SortedDocValues() {
+
+ @Override
+ public int getOrd(int docID) {
+ return (int) ordinals.get(docID);
+ }
+
+ @Override
+ public BytesRef lookupOrd(int ord) {
+ return binary.get(ord);
+ }
+
+ @Override
+ public int getValueCount() {
+ return valueCount;
+ }
+
+ @Override
+ public int lookupTerm(BytesRef key) {
+ if (binary instanceof CompressedBinaryDocValues) {
+ return (int) ((CompressedBinaryDocValues)binary).lookupTerm(key);
+ } else {
+ return super.lookupTerm(key);
+ }
+ }
+
+ @Override
+ public TermsEnum termsEnum() {
+ if (binary instanceof CompressedBinaryDocValues) {
+ return ((CompressedBinaryDocValues)binary).getTermsEnum();
+ } else {
+ return super.termsEnum();
+ }
+ }
+ };
+ }
+
+ /** returns an address instance for sortedset ordinal lists */
+ private MonotonicBlockPackedReader getOrdIndexInstance(IndexInput data, FieldInfo field, NumericEntry entry) throws IOException {
+ final MonotonicBlockPackedReader ordIndex;
+ synchronized (ordIndexInstances) {
+ MonotonicBlockPackedReader ordIndexInstance = ordIndexInstances.get(field.number);
+ if (ordIndexInstance == null) {
+ data.seek(entry.offset);
+ ordIndexInstance = MonotonicBlockPackedReader.of(data, entry.packedIntsVersion, entry.blockSize, entry.count+1, false);
+ ordIndexInstances.put(field.number, ordIndexInstance);
+ ramBytesUsed.addAndGet(ordIndexInstance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
+ }
+ ordIndex = ordIndexInstance;
+ }
+ return ordIndex;
+ }
+
+ @Override
+ public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
+ SortedSetEntry ss = sortedNumerics.get(field.number);
+ NumericEntry numericEntry = numerics.get(field.number);
+ final LongValues values = getNumeric(numericEntry);
+ if (ss.format == SORTED_SINGLE_VALUED) {
+ final Bits docsWithField = getMissingBits(numericEntry.missingOffset);
+ return DocValues.singleton(values, docsWithField);
+ } else if (ss.format == SORTED_WITH_ADDRESSES) {
+ final IndexInput data = this.data.clone();
+ final MonotonicBlockPackedReader ordIndex = getOrdIndexInstance(data, field, ordIndexes.get(field.number));
+
+ return new SortedNumericDocValues() {
+ long startOffset;
+ long endOffset;
+
+ @Override
+ public void setDocument(int doc) {
+ startOffset = ordIndex.get(doc);
+ endOffset = ordIndex.get(doc+1L);
+ }
+
+ @Override
+ public long valueAt(int index) {
+ return values.get(startOffset + index);
+ }
+
+ @Override
+ public int count() {
+ return (int) (endOffset - startOffset);
+ }
+ };
+ } else {
+ throw new AssertionError();
+ }
+ }
+
+ @Override
+ public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
+ SortedSetEntry ss = sortedSets.get(field.number);
+ if (ss.format == SORTED_SINGLE_VALUED) {
+ final SortedDocValues values = getSorted(field);
+ return DocValues.singleton(values);
+ } else if (ss.format != SORTED_WITH_ADDRESSES) {
+ throw new AssertionError();
+ }
+
+ final IndexInput data = this.data.clone();
+ final long valueCount = binaries.get(field.number).count;
+ // we keep the byte[]s and list of ords on disk, these could be large
+ final LongBinaryDocValues binary = (LongBinaryDocValues) getBinary(field);
+ final LongValues ordinals = getNumeric(ords.get(field.number));
+ // but the addresses to the ord stream are in RAM
+ final MonotonicBlockPackedReader ordIndex = getOrdIndexInstance(data, field, ordIndexes.get(field.number));
+
+ return new RandomAccessOrds() {
+ long startOffset;
+ long offset;
+ long endOffset;
+
+ @Override
+ public long nextOrd() {
+ if (offset == endOffset) {
+ return NO_MORE_ORDS;
+ } else {
+ long ord = ordinals.get(offset);
+ offset++;
+ return ord;
+ }
+ }
+
+ @Override
+ public void setDocument(int docID) {
+ startOffset = offset = ordIndex.get(docID);
+ endOffset = ordIndex.get(docID+1L);
+ }
+
+ @Override
+ public BytesRef lookupOrd(long ord) {
+ return binary.get(ord);
+ }
+
+ @Override
+ public long getValueCount() {
+ return valueCount;
+ }
+
+ @Override
+ public long lookupTerm(BytesRef key) {
+ if (binary instanceof CompressedBinaryDocValues) {
+ return ((CompressedBinaryDocValues)binary).lookupTerm(key);
+ } else {
+ return super.lookupTerm(key);
+ }
+ }
+
+ @Override
+ public TermsEnum termsEnum() {
+ if (binary instanceof CompressedBinaryDocValues) {
+ return ((CompressedBinaryDocValues)binary).getTermsEnum();
+ } else {
+ return super.termsEnum();
+ }
+ }
+
+ @Override
+ public long ordAt(int index) {
+ return ordinals.get(startOffset + index);
+ }
+
+ @Override
+ public int cardinality() {
+ return (int) (endOffset - startOffset);
+ }
+ };
+ }
+
+ private Bits getMissingBits(final long offset) throws IOException {
+ if (offset == -1) {
+ return new Bits.MatchAllBits(maxDoc);
+ } else {
+ int length = (int) ((maxDoc + 7L) >>> 3);
+ final RandomAccessInput in = data.randomAccessSlice(offset, length);
+ return new Bits() {
+ @Override
+ public boolean get(int index) {
+ try {
+ return (in.readByte(index >> 3) & (1 << (index & 7))) != 0;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public int length() {
+ return maxDoc;
+ }
+ };
+ }
+ }
+
+ @Override
+ public Bits getDocsWithField(FieldInfo field) throws IOException {
+ switch(field.getDocValuesType()) {
+ case SORTED_SET:
+ return DocValues.docsWithValue(getSortedSet(field), maxDoc);
+ case SORTED_NUMERIC:
+ return DocValues.docsWithValue(getSortedNumeric(field), maxDoc);
+ case SORTED:
+ return DocValues.docsWithValue(getSorted(field), maxDoc);
+ case BINARY:
+ BinaryEntry be = binaries.get(field.number);
+ return getMissingBits(be.missingOffset);
+ case NUMERIC:
+ NumericEntry ne = numerics.get(field.number);
+ return getMissingBits(ne.missingOffset);
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ data.close();
+ }
+
+ /** metadata entry for a numeric docvalues field */
+ static class NumericEntry {
+ private NumericEntry() {}
+ /** offset to the bitset representing docsWithField, or -1 if no documents have missing values */
+ long missingOffset;
+ /** offset to the actual numeric values */
+ public long offset;
+ /** end offset to the actual numeric values */
+ public long endOffset;
+ /** bits per value used to pack the numeric values */
+ public int bitsPerValue;
+
+ int format;
+ /** packed ints version used to encode these numerics */
+ public int packedIntsVersion;
+ /** count of values written */
+ public long count;
+ /** packed ints blocksize */
+ public int blockSize;
+
+ long minValue;
+ long gcd;
+ long table[];
+ }
+
+ /** metadata entry for a binary docvalues field */
+ static class BinaryEntry {
+ private BinaryEntry() {}
+ /** offset to the bitset representing docsWithField, or -1 if no documents have missing values */
+ long missingOffset;
+ /** offset to the actual binary values */
+ long offset;
+
+ int format;
+ /** count of values written */
+ public long count;
+ int minLength;
+ int maxLength;
+ /** offset to the addressing data that maps a value to its slice of the byte[] */
+ public long addressesOffset;
+ /** offset to the reverse index */
+ public long reverseIndexOffset;
+ /** packed ints version used to encode addressing information */
+ public int packedIntsVersion;
+ /** packed ints blocksize */
+ public int blockSize;
+ }
+
+ /** metadata entry for a sorted-set docvalues field */
+ static class SortedSetEntry {
+ private SortedSetEntry() {}
+ int format;
+ }
+
+ // internally we compose complex dv (sorted/sortedset) from other ones
+ static abstract class LongBinaryDocValues extends BinaryDocValues {
+ @Override
+ public final BytesRef get(int docID) {
+ return get((long)docID);
+ }
+
+ abstract BytesRef get(long id);
+ }
+
+ // used for reverse lookup to a small range of blocks
+ static class ReverseTermsIndex {
+ public MonotonicBlockPackedReader termAddresses;
+ public PagedBytes.Reader terms;
+ }
+
+ //in the compressed case, we add a few additional operations for
+ //more efficient reverse lookup and enumeration
+ static final class CompressedBinaryDocValues extends LongBinaryDocValues {
+ final long numValues;
+ final long numIndexValues;
+ final int maxTermLength;
+ final MonotonicBlockPackedReader addresses;
+ final IndexInput data;
+ final CompressedBinaryTermsEnum termsEnum;
+ final PagedBytes.Reader reverseTerms;
+ final MonotonicBlockPackedReader reverseAddresses;
+ final long numReverseIndexValues;
+
+ public CompressedBinaryDocValues(BinaryEntry bytes, MonotonicBlockPackedReader addresses, ReverseTermsIndex index, IndexInput data) throws IOException {
+ this.maxTermLength = bytes.maxLength;
+ this.numValues = bytes.count;
+ this.addresses = addresses;
+ this.numIndexValues = addresses.size();
+ this.data = data;
+ this.reverseTerms = index.terms;
+ this.reverseAddresses = index.termAddresses;
+ this.numReverseIndexValues = reverseAddresses.size();
+ this.termsEnum = getTermsEnum(data);
+ }
+
+ @Override
+ public BytesRef get(long id) {
+ try {
+ termsEnum.seekExact(id);
+ return termsEnum.term();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ long lookupTerm(BytesRef key) {
+ try {
+ switch (termsEnum.seekCeil(key)) {
+ case FOUND: return termsEnum.ord();
+ case NOT_FOUND: return -termsEnum.ord()-1;
+ default: return -numValues-1;
+ }
+ } catch (IOException bogus) {
+ throw new RuntimeException(bogus);
+ }
+ }
+
+ TermsEnum getTermsEnum() {
+ try {
+ return getTermsEnum(data.clone());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private CompressedBinaryTermsEnum getTermsEnum(IndexInput input) throws IOException {
+ return new CompressedBinaryTermsEnum(input);
+ }
+
+ class CompressedBinaryTermsEnum extends TermsEnum {
+ private long currentOrd = -1;
+ // offset to the start of the current block
+ private long currentBlockStart;
+ private final IndexInput input;
+ // delta from currentBlockStart to start of each term
+ private final int offsets[] = new int[INTERVAL_COUNT];
+ private final byte buffer[] = new byte[2*INTERVAL_COUNT-1];
+
+ private final BytesRef term = new BytesRef(maxTermLength);
+ private final BytesRef firstTerm = new BytesRef(maxTermLength);
+ private final BytesRef scratch = new BytesRef();
+
+ CompressedBinaryTermsEnum(IndexInput input) throws IOException {
+ this.input = input;
+ input.seek(0);
+ }
+
+ private void readHeader() throws IOException {
+ firstTerm.length = input.readVInt();
+ input.readBytes(firstTerm.bytes, 0, firstTerm.length);
+ input.readBytes(buffer, 0, INTERVAL_COUNT-1);
+ if (buffer[0] == -1) {
+ readShortAddresses();
+ } else {
+ readByteAddresses();
+ }
+ currentBlockStart = input.getFilePointer();
+ }
+
+ // read single byte addresses: each is delta - 2
+ // (shared prefix byte and length > 0 are both implicit)
+ private void readByteAddresses() throws IOException {
+ int addr = 0;
+ for (int i = 1; i < offsets.length; i++) {
+ addr += 2 + (buffer[i-1] & 0xFF);
+ offsets[i] = addr;
+ }
+ }
+
+ // read double byte addresses: each is delta - 2
+ // (shared prefix byte and length > 0 are both implicit)
+ private void readShortAddresses() throws IOException {
+ input.readBytes(buffer, INTERVAL_COUNT-1, INTERVAL_COUNT);
+ int addr = 0;
+ for (int i = 1; i < offsets.length; i++) {
+ int x = i<<1;
+ addr += 2 + ((buffer[x-1] << 8) | (buffer[x] & 0xFF));
+ offsets[i] = addr;
+ }
+ }
+
+ // set term to the first term
+ private void readFirstTerm() throws IOException {
+ term.length = firstTerm.length;
+ System.arraycopy(firstTerm.bytes, firstTerm.offset, term.bytes, 0, term.length);
+ }
+
+ // read term at offset, delta encoded from first term
+ private void readTerm(int offset) throws IOException {
+ int start = input.readByte() & 0xFF;
+ System.arraycopy(firstTerm.bytes, firstTerm.offset, term.bytes, 0, start);
+ int suffix = offsets[offset] - offsets[offset-1] - 1;
+ input.readBytes(term.bytes, start, suffix);
+ term.length = start + suffix;
+ }
+
+ @Override
+ public BytesRef next() throws IOException {
+ currentOrd++;
+ if (currentOrd >= numValues) {
+ return null;
+ } else {
+ int offset = (int) (currentOrd & INTERVAL_MASK);
+ if (offset == 0) {
+ // switch to next block
+ readHeader();
+ readFirstTerm();
+ } else {
+ readTerm(offset);
+ }
+ return term;
+ }
+ }
+
+ // binary search reverse index to find smaller
+ // range of blocks to search
+ long binarySearchIndex(BytesRef text) throws IOException {
+ long low = 0;
+ long high = numReverseIndexValues - 1;
+ while (low <= high) {
+ long mid = (low + high) >>> 1;
+ reverseTerms.fill(scratch, reverseAddresses.get(mid));
+ int cmp = scratch.compareTo(text);
+
+ if (cmp < 0) {
+ low = mid + 1;
+ } else if (cmp > 0) {
+ high = mid - 1;
+ } else {
+ return mid;
+ }
+ }
+ return high;
+ }
+
+ // binary search against first term in block range
+ // to find term's block
+ long binarySearchBlock(BytesRef text, long low, long high) throws IOException {
+ while (low <= high) {
+ long mid = (low + high) >>> 1;
+ input.seek(addresses.get(mid));
+ term.length = input.readVInt();
+ input.readBytes(term.bytes, 0, term.length);
+ int cmp = term.compareTo(text);
+
+ if (cmp < 0) {
+ low = mid + 1;
+ } else if (cmp > 0) {
+ high = mid - 1;
+ } else {
+ return mid;
+ }
+ }
+ return high;
+ }
+
+ @Override
+ public SeekStatus seekCeil(BytesRef text) throws IOException {
+ // locate block: narrow to block range with index, then search blocks
+ final long block;
+ long indexPos = binarySearchIndex(text);
+ if (indexPos < 0) {
+ block = 0;
+ } else {
+ long low = indexPos << BLOCK_INTERVAL_SHIFT;
+ long high = Math.min(numIndexValues - 1, low + BLOCK_INTERVAL_MASK);
+ block = Math.max(low, binarySearchBlock(text, low, high));
+ }
+
+ // position before block, then scan to term.
+ input.seek(addresses.get(block));
+ currentOrd = (block << INTERVAL_SHIFT) - 1;
+
+ while (next() != null) {
+ int cmp = term.compareTo(text);
+ if (cmp == 0) {
+ return SeekStatus.FOUND;
+ } else if (cmp > 0) {
+ return SeekStatus.NOT_FOUND;
+ }
+ }
+ return SeekStatus.END;
+ }
+
+ @Override
+ public void seekExact(long ord) throws IOException {
+ long block = ord >>> INTERVAL_SHIFT;
+ if (block != currentOrd >>> INTERVAL_SHIFT) {
+ // switch to different block
+ input.seek(addresses.get(block));
+ readHeader();
+ }
+
+ currentOrd = ord;
+
+ int offset = (int) (ord & INTERVAL_MASK);
+ if (offset == 0) {
+ readFirstTerm();
+ } else {
+ input.seek(currentBlockStart + offsets[offset-1]);
+ readTerm(offset);
+ }
+ }
+
+ @Override
+ public BytesRef term() throws IOException {
+ return term;
+ }
+
+ @Override
+ public long ord() throws IOException {
+ return currentOrd;
+ }
+
+ @Override
+ public int docFreq() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long totalTermFreq() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+ }
+ }
+}
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/package.html?rev=1617975&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/package.html (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/package.html Thu Aug 14 15:30:44 2014
@@ -0,0 +1,404 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+Lucene 4.10 file format.
+
+<h1>Apache Lucene - Index File Formats</h1>
+<div>
+<ul>
+<li><a href="#Introduction">Introduction</a></li>
+<li><a href="#Definitions">Definitions</a>
+ <ul>
+ <li><a href="#Inverted_Indexing">Inverted Indexing</a></li>
+ <li><a href="#Types_of_Fields">Types of Fields</a></li>
+ <li><a href="#Segments">Segments</a></li>
+ <li><a href="#Document_Numbers">Document Numbers</a></li>
+ </ul>
+</li>
+<li><a href="#Overview">Index Structure Overview</a></li>
+<li><a href="#File_Naming">File Naming</a></li>
+<li><a href="#file-names">Summary of File Extensions</a></li>
+ <ul>
+ <li><a href="#Lock_File">Lock File</a></li>
+ <li><a href="#History">History</a></li>
+ <li><a href="#Limitations">Limitations</a></li>
+ </ul>
+</ul>
+</div>
+<a name="Introduction"></a>
+<h2>Introduction</h2>
+<div>
+<p>This document defines the index file formats used in this version of Lucene.
+If you are using a different version of Lucene, please consult the copy of
+<code>docs/</code> that was distributed with
+the version you are using.</p>
+<p>Apache Lucene is written in Java, but several efforts are underway to write
+<a href="http://wiki.apache.org/lucene-java/LuceneImplementations">versions of
+Lucene in other programming languages</a>. If these versions are to remain
+compatible with Apache Lucene, then a language-independent definition of the
+Lucene index format is required. This document thus attempts to provide a
+complete and independent definition of the Apache Lucene file formats.</p>
+<p>As Lucene evolves, this document should evolve. Versions of Lucene in
+different programming languages should endeavor to agree on file formats, and
+generate new versions of this document.</p>
+</div>
+<a name="Definitions" id="Definitions"></a>
+<h2>Definitions</h2>
+<div>
+<p>The fundamental concepts in Lucene are index, document, field and term.</p>
+<p>An index contains a sequence of documents.</p>
+<ul>
+<li>A document is a sequence of fields.</li>
+<li>A field is a named sequence of terms.</li>
+<li>A term is a sequence of bytes.</li>
+</ul>
+<p>The same sequence of bytes in two different fields is considered a different
+term. Thus terms are represented as a pair: the string naming the field, and the
+bytes within the field.</p>
+<a name="Inverted_Indexing"></a>
+<h3>Inverted Indexing</h3>
+<p>The index stores statistics about terms in order to make term-based search
+more efficient. Lucene's index falls into the family of indexes known as an
+<i>inverted index.</i> This is because it can list, for a term, the documents
+that contain it. This is the inverse of the natural relationship, in which
+documents list terms.</p>
+<a name="Types_of_Fields"></a>
+<h3>Types of Fields</h3>
+<p>In Lucene, fields may be <i>stored</i>, in which case their text is stored
+in the index literally, in a non-inverted manner. Fields that are inverted are
+called <i>indexed</i>. A field may be both stored and indexed.</p>
+<p>The text of a field may be <i>tokenized</i> into terms to be indexed, or the
+text of a field may be used literally as a term to be indexed. Most fields are
+tokenized, but sometimes it is useful for certain identifier fields to be
+indexed literally.</p>
+<p>See the {@link org.apache.lucene.document.Field Field}
+java docs for more information on Fields.</p>
+<a name="Segments" id="Segments"></a>
+<h3>Segments</h3>
+<p>Lucene indexes may be composed of multiple sub-indexes, or <i>segments</i>.
+Each segment is a fully independent index, which could be searched separately.
+Indexes evolve by:</p>
+<ol>
+<li>Creating new segments for newly added documents.</li>
+<li>Merging existing segments.</li>
+</ol>
+<p>Searches may involve multiple segments and/or multiple indexes, each index
+potentially composed of a set of segments.</p>
+<a name="Document_Numbers"></a>
+<h3>Document Numbers</h3>
+<p>Internally, Lucene refers to documents by an integer <i>document number</i>.
+The first document added to an index is numbered zero, and each subsequent
+document added gets a number one greater than the previous.</p>
+<p>Note that a document's number may change, so caution should be taken when
+storing these numbers outside of Lucene. In particular, numbers may change in
+the following situations:</p>
+<ul>
+<li>
+<p>The numbers stored in each segment are unique only within the segment, and
+must be converted before they can be used in a larger context. The standard
+technique is to allocate each segment a range of values, based on the range of
+numbers used in that segment. To convert a document number from a segment to an
+external value, the segment's <i>base</i> document number is added. To convert
+an external value back to a segment-specific value, the segment is identified
+by the range that the external value is in, and the segment's base value is
+subtracted. For example two five document segments might be combined, so that
+the first segment has a base value of zero, and the second of five. Document
+three from the second segment would have an external value of eight.</p>
+</li>
+<li>
+<p>When documents are deleted, gaps are created in the numbering. These are
+eventually removed as the index evolves through merging. Deleted documents are
+dropped when segments are merged. A freshly-merged segment thus has no gaps in
+its numbering.</p>
+</li>
+</ul>
+</div>
+<a name="Overview" id="Overview"></a>
+<h2>Index Structure Overview</h2>
+<div>
+<p>Each segment index maintains the following:</p>
+<ul>
+<li>
+{@link org.apache.lucene.codecs.lucene46.Lucene46SegmentInfoFormat Segment info}.
+ This contains metadata about a segment, such as the number of documents,
+ what files it uses,
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene46.Lucene46FieldInfosFormat Field names}.
+ This contains the set of field names used in the index.
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Stored Field values}.
+This contains, for each document, a list of attribute-value pairs, where the attributes
+are field names. These are used to store auxiliary information about the document, such as
+its title, url, or an identifier to access a database. The set of stored fields are what is
+returned for each hit when searching. This is keyed by document number.
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term dictionary}.
+A dictionary containing all of the terms used in all of the
+indexed fields of all of the documents. The dictionary also contains the number
+of documents which contain the term, and pointers to the term's frequency and
+proximity data.
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Frequency data}.
+For each term in the dictionary, the numbers of all the
+documents that contain that term, and the frequency of the term in that
+document, unless frequencies are omitted (IndexOptions.DOCS_ONLY)
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Proximity data}.
+For each term in the dictionary, the positions that the
+term occurs in each document. Note that this will not exist if all fields in
+all documents omit position data.
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene49.Lucene49NormsFormat Normalization factors}.
+For each field in each document, a value is stored
+that is multiplied into the score for hits on that field.
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat Term Vectors}.
+For each field in each document, the term vector (sometimes
+called document vector) may be stored. A term vector consists of term text and
+term frequency. To add Term Vectors to your index see the
+{@link org.apache.lucene.document.Field Field} constructors
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene410.Lucene410DocValuesFormat Per-document values}.
+Like stored values, these are also keyed by document
+number, but are generally intended to be loaded into main memory for fast
+access. Whereas stored values are generally intended for summary results from
+searches, per-document values are useful for things like scoring factors.
+</li>
+<li>
+{@link org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat Deleted documents}.
+An optional file indicating which documents are deleted.
+</li>
+</ul>
+<p>Details on each of these are provided in their linked pages.</p>
+</div>
+<a name="File_Naming"></a>
+<h2>File Naming</h2>
+<div>
+<p>All files belonging to a segment have the same name with varying extensions.
+The extensions correspond to the different file formats described below. When
+using the Compound File format (default in 1.4 and greater) these files (except
+for the Segment info file, the Lock file, and Deleted documents file) are collapsed
+into a single .cfs file (see below for details)</p>
+<p>Typically, all segments in an index are stored in a single directory,
+although this is not required.</p>
+<p>As of version 2.1 (lock-less commits), file names are never re-used (there
+is one exception, "segments.gen", see below). That is, when any file is saved
+to the Directory it is given a never before used filename. This is achieved
+using a simple generations approach. For example, the first segments file is
+segments_1, then segments_2, etc. The generation is a sequential long integer
+represented in alpha-numeric (base 36) form.</p>
+</div>
+<a name="file-names" id="file-names"></a>
+<h2>Summary of File Extensions</h2>
+<div>
+<p>The following table summarizes the names and extensions of the files in
+Lucene:</p>
+<table cellspacing="1" cellpadding="4">
+<tr>
+<th>Name</th>
+<th>Extension</th>
+<th>Brief Description</th>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.index.SegmentInfos Segments File}</td>
+<td>segments.gen, segments_N</td>
+<td>Stores information about a commit point</td>
+</tr>
+<tr>
+<td><a href="#Lock_File">Lock File</a></td>
+<td>write.lock</td>
+<td>The Write lock prevents multiple IndexWriters from writing to the same
+file.</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat Segment Info}</td>
+<td>.si</td>
+<td>Stores metadata about a segment</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.store.CompoundFileDirectory Compound File}</td>
+<td>.cfs, .cfe</td>
+<td>An optional "virtual" file consisting of all the other index files for
+systems that frequently run out of file handles.</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene46.Lucene46FieldInfosFormat Fields}</td>
+<td>.fnm</td>
+<td>Stores information about the fields</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Field Index}</td>
+<td>.fdx</td>
+<td>Contains pointers to field data</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Field Data}</td>
+<td>.fdt</td>
+<td>The stored fields for documents</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Dictionary}</td>
+<td>.tim</td>
+<td>The term dictionary, stores term info</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Index}</td>
+<td>.tip</td>
+<td>The index into the Term Dictionary</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Frequencies}</td>
+<td>.doc</td>
+<td>Contains the list of docs which contain each term along with frequency</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Positions}</td>
+<td>.pos</td>
+<td>Stores position information about where a term occurs in the index</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Payloads}</td>
+<td>.pay</td>
+<td>Stores additional per-position metadata information such as character offsets and user payloads</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene49.Lucene49NormsFormat Norms}</td>
+<td>.nvd, .nvm</td>
+<td>Encodes length and boost factors for docs and fields</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene410.Lucene410DocValuesFormat Per-Document Values}</td>
+<td>.dvd, .dvm</td>
+<td>Encodes additional scoring factors or other per-document information.</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat Term Vector Index}</td>
+<td>.tvx</td>
+<td>Stores offset into the document data file</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat Term Vector Documents}</td>
+<td>.tvd</td>
+<td>Contains information about each document that has term vectors</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat Term Vector Fields}</td>
+<td>.tvf</td>
+<td>The field level info about term vectors</td>
+</tr>
+<tr>
+<td>{@link org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat Deleted Documents}</td>
+<td>.del</td>
+<td>Info about what files are deleted</td>
+</tr>
+</table>
+</div>
+<a name="Lock_File" id="Lock_File"></a>
+<h2>Lock File</h2>
+The write lock, which is stored in the index directory by default, is named
+"write.lock". If the lock directory is different from the index directory then
+the write lock will be named "XXXX-write.lock" where XXXX is a unique prefix
+derived from the full path to the index directory. When this file is present, a
+writer is currently modifying the index (adding or removing documents). This
+lock file ensures that only one writer is modifying the index at a time.</p>
+<a name="History"></a>
+<h2>History</h2>
+<p>Compatibility notes are provided in this document, describing how file
+formats have changed from prior versions:</p>
+<ul>
+<li>In version 2.1, the file format was changed to allow lock-less commits (ie,
+no more commit lock). The change is fully backwards compatible: you can open a
+pre-2.1 index for searching or adding/deleting of docs. When the new segments
+file is saved (committed), it will be written in the new file format (meaning
+no specific "upgrade" process is needed). But note that once a commit has
+occurred, pre-2.1 Lucene will not be able to read the index.</li>
+<li>In version 2.3, the file format was changed to allow segments to share a
+single set of doc store (vectors & stored fields) files. This allows for
+faster indexing in certain cases. The change is fully backwards compatible (in
+the same way as the lock-less commits change in 2.1).</li>
+<li>In version 2.4, Strings are now written as true UTF-8 byte sequence, not
+Java's modified UTF-8. See <a href="http://issues.apache.org/jira/browse/LUCENE-510">
+LUCENE-510</a> for details.</li>
+<li>In version 2.9, an optional opaque Map<String,String> CommitUserData
+may be passed to IndexWriter's commit methods (and later retrieved), which is
+recorded in the segments_N file. See <a href="http://issues.apache.org/jira/browse/LUCENE-1382">
+LUCENE-1382</a> for details. Also,
+diagnostics were added to each segment written recording details about why it
+was written (due to flush, merge; which OS/JRE was used; etc.). See issue
+<a href="http://issues.apache.org/jira/browse/LUCENE-1654">LUCENE-1654</a> for details.</li>
+<li>In version 3.0, compressed fields are no longer written to the index (they
+can still be read, but on merge the new segment will write them, uncompressed).
+See issue <a href="http://issues.apache.org/jira/browse/LUCENE-1960">LUCENE-1960</a>
+for details.</li>
+<li>In version 3.1, segments records the code version that created them. See
+<a href="http://issues.apache.org/jira/browse/LUCENE-2720">LUCENE-2720</a> for details.
+Additionally segments track explicitly whether or not they have term vectors.
+See <a href="http://issues.apache.org/jira/browse/LUCENE-2811">LUCENE-2811</a>
+for details.</li>
+<li>In version 3.2, numeric fields are written as natively to stored fields
+file, previously they were stored in text format only.</li>
+<li>In version 3.4, fields can omit position data while still indexing term
+frequencies.</li>
+<li>In version 4.0, the format of the inverted index became extensible via
+the {@link org.apache.lucene.codecs.Codec Codec} api. Fast per-document storage
+({@code DocValues}) was introduced. Normalization factors need no longer be a
+single byte, they can be any {@link org.apache.lucene.index.NumericDocValues NumericDocValues}.
+Terms need not be unicode strings, they can be any byte sequence. Term offsets
+can optionally be indexed into the postings lists. Payloads can be stored in the
+term vectors.</li>
+<li>In version 4.1, the format of the postings list changed to use either
+of FOR compression or variable-byte encoding, depending upon the frequency
+of the term. Terms appearing only once were changed to inline directly into
+the term dictionary. Stored fields are compressed by default. </li>
+<li>In version 4.2, term vectors are compressed by default. DocValues has
+a new multi-valued type (SortedSet), that can be used for faceting/grouping/joining
+on multi-valued fields.</li>
+<li>In version 4.5, DocValues were extended to explicitly represent missing values.</li>
+<li>In version 4.6, FieldInfos were extended to support per-field DocValues generation, to
+allow updating NumericDocValues fields.</li>
+<li>In version 4.8, checksum footers were added to the end of each index file
+for improved data integrity. Specifically, the last 8 bytes of every index file
+contain the zlib-crc32 checksum of the file.</li>
+<li>In version 4.9, DocValues has a new multi-valued numeric type (SortedNumeric)
+that is suitable for faceting/sorting/analytics.
+</li>
+</ul>
+<a name="Limitations" id="Limitations"></a>
+<h2>Limitations</h2>
+<div>
+<p>Lucene uses a Java <code>int</code> to refer to
+document numbers, and the index file format uses an <code>Int32</code>
+on-disk to store document numbers. This is a limitation
+of both the index file format and the current implementation. Eventually these
+should be replaced with either <code>UInt64</code> values, or
+better yet, {@link org.apache.lucene.store.DataOutput#writeVInt VInt} values which have no limit.</p>
+</div>
+</body>
+</html>
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49Codec.java?rev=1617975&r1=1617974&r2=1617975&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49Codec.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49Codec.java Thu Aug 14 15:30:44 2014
@@ -17,7 +17,10 @@ package org.apache.lucene.codecs.lucene4
* limitations under the License.
*/
+import java.io.IOException;
+
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FilterCodec;
@@ -35,6 +38,7 @@ import org.apache.lucene.codecs.lucene46
import org.apache.lucene.codecs.lucene46.Lucene46SegmentInfoFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
+import org.apache.lucene.index.SegmentWriteState;
/**
* Implements the Lucene 4.9 index format, with configurable per-field postings
@@ -131,10 +135,15 @@ public class Lucene49Codec extends Codec
private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41");
private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene49");
- private final NormsFormat normsFormat = new Lucene49NormsFormat();
+ private final NormsFormat normsFormat = new Lucene49NormsFormat() {
+ @Override
+ public DocValuesConsumer normsConsumer(SegmentWriteState state) throws IOException {
+ throw new UnsupportedOperationException("this codec can only be used for reading");
+ }
+ };
@Override
- public final NormsFormat normsFormat() {
- return normsFormat;
+ public NormsFormat normsFormat() {
+ return normsFormat;
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java?rev=1617975&r1=1617974&r2=1617975&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java Thu Aug 14 15:30:44 2014
@@ -28,6 +28,7 @@ import org.apache.lucene.codecs.DocValue
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.BytesRef;
@@ -92,6 +93,7 @@ class Lucene49DocValuesConsumer extends
@Override
public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
+ checkCanWrite(field);
addNumericField(field, values, true);
}
@@ -246,6 +248,7 @@ class Lucene49DocValuesConsumer extends
@Override
public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
+ checkCanWrite(field);
// write the byte[] data
meta.writeVInt(field.number);
meta.writeByte(Lucene49DocValuesFormat.BINARY);
@@ -362,6 +365,7 @@ class Lucene49DocValuesConsumer extends
@Override
public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
+ checkCanWrite(field);
meta.writeVInt(field.number);
meta.writeByte(Lucene49DocValuesFormat.SORTED);
addTermsDict(field, values);
@@ -370,6 +374,7 @@ class Lucene49DocValuesConsumer extends
@Override
public void addSortedNumericField(FieldInfo field, final Iterable<Number> docToValueCount, final Iterable<Number> values) throws IOException {
+ checkCanWrite(field);
meta.writeVInt(field.number);
meta.writeByte(Lucene49DocValuesFormat.SORTED_NUMERIC);
if (isSingleValued(docToValueCount)) {
@@ -387,6 +392,7 @@ class Lucene49DocValuesConsumer extends
@Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, final Iterable<Number> docToOrdCount, final Iterable<Number> ords) throws IOException {
+ checkCanWrite(field);
meta.writeVInt(field.number);
meta.writeByte(Lucene49DocValuesFormat.SORTED_SET);
@@ -452,4 +458,14 @@ class Lucene49DocValuesConsumer extends
meta = data = null;
}
}
+
+ void checkCanWrite(FieldInfo field) {
+ if ((field.getDocValuesType() == DocValuesType.NUMERIC ||
+ field.getDocValuesType() == DocValuesType.BINARY) &&
+ field.getDocValuesGen() != -1) {
+ // ok
+ } else {
+ throw new UnsupportedOperationException("this codec can only be used for reading");
+ }
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesFormat.java?rev=1617975&r1=1617974&r2=1617975&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesFormat.java Thu Aug 14 15:30:44 2014
@@ -164,7 +164,7 @@ import org.apache.lucene.util.packed.Mon
* </ol>
* @lucene.experimental
*/
-public final class Lucene49DocValuesFormat extends DocValuesFormat {
+public class Lucene49DocValuesFormat extends DocValuesFormat {
/** Sole Constructor */
public Lucene49DocValuesFormat() {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java?rev=1617975&r1=1617974&r2=1617975&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java Thu Aug 14 15:30:44 2014
@@ -97,7 +97,7 @@ import org.apache.lucene.util.packed.Pac
* </ol>
* @lucene.experimental
*/
-public final class Lucene49NormsFormat extends NormsFormat {
+public class Lucene49NormsFormat extends NormsFormat {
/** Sole Constructor */
public Lucene49NormsFormat() {}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/package.html?rev=1617975&r1=1617974&r2=1617975&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/package.html (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/package.html Thu Aug 14 15:30:44 2014
@@ -61,13 +61,13 @@ name of your codec.
If you just want to customise the {@link org.apache.lucene.codecs.PostingsFormat}, or use different postings
formats for different fields, then you can register your custom postings format in the same way (in
META-INF/services/org.apache.lucene.codecs.PostingsFormat), and then extend the default
- {@link org.apache.lucene.codecs.lucene49.Lucene49Codec} and override
- {@link org.apache.lucene.codecs.lucene49.Lucene49Codec#getPostingsFormatForField(String)} to return your custom
+ {@link org.apache.lucene.codecs.lucene410.Lucene410Codec} and override
+ {@link org.apache.lucene.codecs.lucene410.Lucene410Codec#getPostingsFormatForField(String)} to return your custom
postings format.
</p>
<p>
Similarly, if you just want to customise the {@link org.apache.lucene.codecs.DocValuesFormat} per-field, have
- a look at {@link org.apache.lucene.codecs.lucene49.Lucene49Codec#getDocValuesFormatForField(String)}.
+ a look at {@link org.apache.lucene.codecs.lucene410.Lucene410Codec#getDocValuesFormatForField(String)}.
</p>
</body>
</html>
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/StringHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/StringHelper.java?rev=1617975&r1=1617974&r2=1617975&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/StringHelper.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/StringHelper.java Thu Aug 14 15:30:44 2014
@@ -46,6 +46,23 @@ public abstract class StringHelper {
return i;
return len;
}
+
+ /**
+ * Returns the length of {@code currentTerm} needed for use as a sort key.
+ * so that {@link BytesRef#compareTo(BytesRef)} still returns the same result.
+ * This method assumes currentTerm comes after priorTerm.
+ */
+ public static int sortKeyLength(final BytesRef priorTerm, final BytesRef currentTerm) {
+ final int currentTermOffset = currentTerm.offset;
+ final int priorTermOffset = priorTerm.offset;
+ final int limit = Math.min(priorTerm.length, currentTerm.length);
+ for (int i = 0; i < limit; i++) {
+ if (priorTerm.bytes[priorTermOffset+i] != currentTerm.bytes[currentTermOffset+i]) {
+ return i+1;
+ }
+ }
+ return Math.min(1+priorTerm.length, currentTerm.length);
+ }
private StringHelper() {
}
Modified: lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec?rev=1617975&r1=1617974&r2=1617975&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec (original)
+++ lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec Thu Aug 14 15:30:44 2014
@@ -19,3 +19,4 @@ org.apache.lucene.codecs.lucene42.Lucene
org.apache.lucene.codecs.lucene45.Lucene45Codec
org.apache.lucene.codecs.lucene46.Lucene46Codec
org.apache.lucene.codecs.lucene49.Lucene49Codec
+org.apache.lucene.codecs.lucene410.Lucene410Codec
Modified: lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat?rev=1617975&r1=1617974&r2=1617975&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat (original)
+++ lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat Thu Aug 14 15:30:44 2014
@@ -16,3 +16,4 @@
org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat
org.apache.lucene.codecs.lucene45.Lucene45DocValuesFormat
org.apache.lucene.codecs.lucene49.Lucene49DocValuesFormat
+org.apache.lucene.codecs.lucene410.Lucene410DocValuesFormat
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java?rev=1617975&r1=1617974&r2=1617975&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java Thu Aug 14 15:30:44 2014
@@ -19,7 +19,7 @@ package org.apache.lucene;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene49.Lucene49Codec;
+import org.apache.lucene.codecs.lucene410.Lucene410Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
@@ -37,7 +37,7 @@ import org.apache.lucene.util.LuceneTest
public class TestExternalCodecs extends LuceneTestCase {
- private static final class CustomPerFieldCodec extends Lucene49Codec {
+ private static final class CustomPerFieldCodec extends Lucene410Codec {
private final PostingsFormat ramFormat = PostingsFormat.forName("RAMOnly");
private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41");
Added: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene410/TestLucene410DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene410/TestLucene410DocValuesFormat.java?rev=1617975&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene410/TestLucene410DocValuesFormat.java (added)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene410/TestLucene410DocValuesFormat.java Thu Aug 14 15:30:44 2014
@@ -0,0 +1,269 @@
+package org.apache.lucene.codecs.lucene410;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.lucene41ords.Lucene41WithOrds;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.BaseCompressingDocValuesFormatTestCase;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.TermsEnum.SeekStatus;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.TestUtil;
+
+/**
+ * Tests Lucene410DocValuesFormat
+ */
+public class TestLucene410DocValuesFormat extends BaseCompressingDocValuesFormatTestCase {
+ private final Codec codec = TestUtil.alwaysDocValuesFormat(new Lucene410DocValuesFormat());
+
+ @Override
+ protected Codec getCodec() {
+ return codec;
+ }
+
+ // TODO: these big methods can easily blow up some of the other ram-hungry codecs...
+ // for now just keep them here, as we want to test this for this format.
+
+ public void testSortedSetVariableLengthBigVsStoredFields() throws Exception {
+ int numIterations = atLeast(1);
+ for (int i = 0; i < numIterations; i++) {
+ doTestSortedSetVsStoredFields(atLeast(300), 1, 32766, 16);
+ }
+ }
+
+ @Nightly
+ public void testSortedSetVariableLengthManyVsStoredFields() throws Exception {
+ int numIterations = atLeast(1);
+ for (int i = 0; i < numIterations; i++) {
+ doTestSortedSetVsStoredFields(TestUtil.nextInt(random(), 1024, 2049), 1, 500, 16);
+ }
+ }
+
+ public void testSortedVariableLengthBigVsStoredFields() throws Exception {
+ int numIterations = atLeast(1);
+ for (int i = 0; i < numIterations; i++) {
+ doTestSortedVsStoredFields(atLeast(300), 1, 32766);
+ }
+ }
+
+ @Nightly
+ public void testSortedVariableLengthManyVsStoredFields() throws Exception {
+ int numIterations = atLeast(1);
+ for (int i = 0; i < numIterations; i++) {
+ doTestSortedVsStoredFields(TestUtil.nextInt(random(), 1024, 2049), 1, 500);
+ }
+ }
+
+ public void testTermsEnumFixedWidth() throws Exception {
+ int numIterations = atLeast(1);
+ for (int i = 0; i < numIterations; i++) {
+ doTestTermsEnumRandom(TestUtil.nextInt(random(), 1025, 5121), 10, 10);
+ }
+ }
+
+ public void testTermsEnumVariableWidth() throws Exception {
+ int numIterations = atLeast(1);
+ for (int i = 0; i < numIterations; i++) {
+ doTestTermsEnumRandom(TestUtil.nextInt(random(), 1025, 5121), 1, 500);
+ }
+ }
+
+ @Nightly
+ public void testTermsEnumRandomMany() throws Exception {
+ int numIterations = atLeast(1);
+ for (int i = 0; i < numIterations; i++) {
+ doTestTermsEnumRandom(TestUtil.nextInt(random(), 1025, 8121), 1, 500);
+ }
+ }
+
+ // TODO: try to refactor this and some termsenum tests into the base class.
+ // to do this we need to fix the test class to get a DVF not a Codec so we can setup
+ // the postings format correctly.
+ private void doTestTermsEnumRandom(int numDocs, int minLength, int maxLength) throws Exception {
+ Directory dir = newFSDirectory(createTempDir());
+ IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));
+ // set to duel against a codec which has ordinals:
+ final PostingsFormat pf = new Lucene41WithOrds();
+ final DocValuesFormat dv = new Lucene410DocValuesFormat();
+ conf.setCodec(new Lucene410Codec() {
+ @Override
+ public PostingsFormat getPostingsFormatForField(String field) {
+ return pf;
+ }
+
+ @Override
+ public DocValuesFormat getDocValuesFormatForField(String field) {
+ return dv;
+ }
+ });
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
+
+ // index some docs
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ Field idField = new StringField("id", Integer.toString(i), Field.Store.NO);
+ doc.add(idField);
+ final int length = TestUtil.nextInt(random(), minLength, maxLength);
+ int numValues = random().nextInt(17);
+ // create a random list of strings
+ List<String> values = new ArrayList<>();
+ for (int v = 0; v < numValues; v++) {
+ values.add(TestUtil.randomSimpleString(random(), minLength, length));
+ }
+
+ // add in any order to the indexed field
+ ArrayList<String> unordered = new ArrayList<>(values);
+ Collections.shuffle(unordered, random());
+ for (String v : values) {
+ doc.add(newStringField("indexed", v, Field.Store.NO));
+ }
+
+ // add in any order to the dv field
+ ArrayList<String> unordered2 = new ArrayList<>(values);
+ Collections.shuffle(unordered2, random());
+ for (String v : unordered2) {
+ doc.add(new SortedSetDocValuesField("dv", new BytesRef(v)));
+ }
+
+ writer.addDocument(doc);
+ if (random().nextInt(31) == 0) {
+ writer.commit();
+ }
+ }
+
+ // delete some docs
+ int numDeletions = random().nextInt(numDocs/10);
+ for (int i = 0; i < numDeletions; i++) {
+ int id = random().nextInt(numDocs);
+ writer.deleteDocuments(new Term("id", Integer.toString(id)));
+ }
+
+ // compare per-segment
+ DirectoryReader ir = writer.getReader();
+ for (AtomicReaderContext context : ir.leaves()) {
+ AtomicReader r = context.reader();
+ Terms terms = r.terms("indexed");
+ if (terms != null) {
+ TermsEnum expected = terms.iterator(null);
+ TermsEnum actual = r.getSortedSetDocValues("dv").termsEnum();
+ assertEquals(r.maxDoc(), expected, actual);
+ }
+ }
+ ir.close();
+
+ writer.forceMerge(1);
+
+ // now compare again after the merge
+ ir = writer.getReader();
+ AtomicReader ar = getOnlySegmentReader(ir);
+ Terms terms = ar.terms("indexed");
+ if (terms != null) {
+ TermsEnum expected = terms.iterator(null);
+ TermsEnum actual = ar.getSortedSetDocValues("dv").termsEnum();
+ assertEquals(ar.maxDoc(), expected, actual);
+ }
+ ir.close();
+
+ writer.close();
+ dir.close();
+ }
+
+ private void assertEquals(long numOrds, TermsEnum expected, TermsEnum actual) throws Exception {
+ BytesRef ref;
+
+ // sequential next() through all terms
+ while ((ref = expected.next()) != null) {
+ assertEquals(ref, actual.next());
+ assertEquals(expected.ord(), actual.ord());
+ assertEquals(expected.term(), actual.term());
+ }
+ assertNull(actual.next());
+
+ // sequential seekExact(ord) through all terms
+ for (long i = 0; i < numOrds; i++) {
+ expected.seekExact(i);
+ actual.seekExact(i);
+ assertEquals(expected.ord(), actual.ord());
+ assertEquals(expected.term(), actual.term());
+ }
+
+ // sequential seekExact(BytesRef) through all terms
+ for (long i = 0; i < numOrds; i++) {
+ expected.seekExact(i);
+ assertTrue(actual.seekExact(expected.term()));
+ assertEquals(expected.ord(), actual.ord());
+ assertEquals(expected.term(), actual.term());
+ }
+
+ // sequential seekCeil(BytesRef) through all terms
+ for (long i = 0; i < numOrds; i++) {
+ expected.seekExact(i);
+ assertEquals(SeekStatus.FOUND, actual.seekCeil(expected.term()));
+ assertEquals(expected.ord(), actual.ord());
+ assertEquals(expected.term(), actual.term());
+ }
+
+ // random seekExact(ord)
+ for (long i = 0; i < numOrds; i++) {
+ long randomOrd = TestUtil.nextLong(random(), 0, numOrds - 1);
+ expected.seekExact(randomOrd);
+ actual.seekExact(randomOrd);
+ assertEquals(expected.ord(), actual.ord());
+ assertEquals(expected.term(), actual.term());
+ }
+
+ // random seekExact(BytesRef)
+ for (long i = 0; i < numOrds; i++) {
+ long randomOrd = TestUtil.nextLong(random(), 0, numOrds - 1);
+ expected.seekExact(randomOrd);
+ actual.seekExact(expected.term());
+ assertEquals(expected.ord(), actual.ord());
+ assertEquals(expected.term(), actual.term());
+ }
+
+ // random seekCeil(BytesRef)
+ for (long i = 0; i < numOrds; i++) {
+ BytesRef target = new BytesRef(TestUtil.randomUnicodeString(random()));
+ SeekStatus expectedStatus = expected.seekCeil(target);
+ assertEquals(expectedStatus, actual.seekCeil(target));
+ if (expectedStatus != SeekStatus.END) {
+ assertEquals(expected.ord(), actual.ord());
+ assertEquals(expected.term(), actual.term());
+ }
+ }
+ }
+}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene49/TestLucene49DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene49/TestLucene49DocValuesFormat.java?rev=1617975&r1=1617974&r2=1617975&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene49/TestLucene49DocValuesFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene49/TestLucene49DocValuesFormat.java Thu Aug 14 15:30:44 2014
@@ -19,14 +19,19 @@ package org.apache.lucene.codecs.lucene4
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseCompressingDocValuesFormatTestCase;
-import org.apache.lucene.util.TestUtil;
+import org.junit.BeforeClass;
/**
* Tests Lucene49DocValuesFormat
*/
public class TestLucene49DocValuesFormat extends BaseCompressingDocValuesFormatTestCase {
- private final Codec codec = TestUtil.alwaysDocValuesFormat(new Lucene49DocValuesFormat());
-
+ private final Codec codec = new Lucene49RWCodec();
+
+ @BeforeClass
+ public static void beforeClass() {
+ OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true; // explicitly instantiates ancient codec
+ }
+
@Override
protected Codec getCodec() {
return codec;
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java?rev=1617975&r1=1617974&r2=1617975&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java Thu Aug 14 15:30:44 2014
@@ -19,15 +19,21 @@ package org.apache.lucene.codecs.lucene4
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseNormsFormatTestCase;
+import org.junit.BeforeClass;
/**
* Tests Lucene49NormsFormat
*/
public class TestLucene49NormsFormat extends BaseNormsFormatTestCase {
- final Codec codec = new Lucene49Codec();
+ private final Codec codec = new Lucene49RWCodec();
+
+ @BeforeClass
+ public static void beforeClass() {
+ OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true; // explicitly instantiates ancient codec
+ }
@Override
protected Codec getCodec() {
return codec;
- }
+ }
}