You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/10/24 04:43:45 UTC
svn commit: r1633991 [3/4] - in /lucene/dev/trunk: ./ dev-tools/ lucene/
lucene/analysis/ lucene/analysis/common/ lucene/backward-codecs/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/blocktree/
lucene/backward-codecs/src/java/org/apache/luc...
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java Fri Oct 24 02:43:41 2014
@@ -71,7 +71,7 @@ final class Lucene50CompoundReader exten
boolean success = false;
handle = directory.openInput(dataFileName, context);
try {
- CodecUtil.checkSegmentHeader(handle, Lucene50CompoundFormat.DATA_CODEC, version, version, si.getId(), "");
+ CodecUtil.checkIndexHeader(handle, Lucene50CompoundFormat.DATA_CODEC, version, version, si.getId(), "");
// NOTE: data file is too costly to verify checksum against all the bytes on open,
// but for now we at least verify proper structure of the checksum footer: which looks
@@ -93,7 +93,7 @@ final class Lucene50CompoundReader exten
try (ChecksumIndexInput entriesStream = dir.openChecksumInput(entriesFileName, IOContext.READONCE)) {
Throwable priorE = null;
try {
- version = CodecUtil.checkSegmentHeader(entriesStream, Lucene50CompoundFormat.ENTRY_CODEC,
+ version = CodecUtil.checkIndexHeader(entriesStream, Lucene50CompoundFormat.ENTRY_CODEC,
Lucene50CompoundFormat.VERSION_START,
Lucene50CompoundFormat.VERSION_CURRENT, segmentID, "");
final int numEntries = entriesStream.readVInt();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java Fri Oct 24 02:43:41 2014
@@ -46,7 +46,7 @@ import org.apache.lucene.store.IndexOutp
* FieldBits,DocValuesBits,DocValuesGen,Attributes> <sup>FieldsCount</sup>,Footer</p>
* <p>Data types:
* <ul>
- * <li>Header --> {@link CodecUtil#checkSegmentHeader SegmentHeader}</li>
+ * <li>Header --> {@link CodecUtil#checkIndexHeader IndexHeader}</li>
* <li>FieldsCount --> {@link DataOutput#writeVInt VInt}</li>
* <li>FieldName --> {@link DataOutput#writeString String}</li>
* <li>FieldBits, IndexOptions, DocValuesBits --> {@link DataOutput#writeByte Byte}</li>
@@ -114,9 +114,9 @@ public final class Lucene50FieldInfosFor
Throwable priorE = null;
FieldInfo infos[] = null;
try {
- CodecUtil.checkSegmentHeader(input, CODEC_NAME,
- FORMAT_START,
- FORMAT_CURRENT,
+ CodecUtil.checkIndexHeader(input, Lucene50FieldInfosFormat.CODEC_NAME,
+ Lucene50FieldInfosFormat.FORMAT_START,
+ Lucene50FieldInfosFormat.FORMAT_CURRENT,
segmentInfo.getId(), segmentSuffix);
final int size = input.readVInt(); //read in the size
@@ -251,7 +251,7 @@ public final class Lucene50FieldInfosFor
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
try (IndexOutput output = directory.createOutput(fileName, context)) {
- CodecUtil.writeSegmentHeader(output, CODEC_NAME, FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
+ CodecUtil.writeIndexHeader(output, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
output.writeVInt(infos.size());
for (FieldInfo fi : infos) {
fi.checkConsistency();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50LiveDocsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50LiveDocsFormat.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50LiveDocsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50LiveDocsFormat.java Fri Oct 24 02:43:41 2014
@@ -41,10 +41,9 @@ import org.apache.lucene.util.MutableBit
* deletions.</p>
* <p>Although per-segment, this file is maintained exterior to compound segment
* files.</p>
- * <p>Deletions (.liv) --> SegmentHeader,Generation,Bits</p>
+ * <p>Deletions (.liv) --> IndexHeader,Generation,Bits</p>
* <ul>
- * <li>SegmentHeader --> {@link CodecUtil#writeSegmentHeader SegmentHeader}</li>
- * <li>Generation --> {@link DataOutput#writeLong Int64}
+ * <li>SegmentHeader --> {@link CodecUtil#writeIndexHeader IndexHeader}</li>
* <li>Bits --> <{@link DataOutput#writeLong Int64}> <sup>LongCount</sup></li>
* </ul>
*/
@@ -85,11 +84,8 @@ public final class Lucene50LiveDocsForma
try (ChecksumIndexInput input = dir.openChecksumInput(name, context)) {
Throwable priorE = null;
try {
- CodecUtil.checkSegmentHeader(input, CODEC_NAME, VERSION_START, VERSION_CURRENT, info.info.getId(), "");
- long filegen = input.readLong();
- if (gen != filegen) {
- throw new CorruptIndexException("file mismatch, expected generation=" + gen + ", got=" + filegen, input);
- }
+ CodecUtil.checkIndexHeader(input, CODEC_NAME, VERSION_START, VERSION_CURRENT,
+ info.info.getId(), Long.toString(gen, Character.MAX_RADIX));
long data[] = new long[FixedBitSet.bits2words(length)];
for (int i = 0; i < data.length; i++) {
data[i] = input.readLong();
@@ -120,8 +116,7 @@ public final class Lucene50LiveDocsForma
}
long data[] = fbs.getBits();
try (IndexOutput output = dir.createOutput(name, context)) {
- CodecUtil.writeSegmentHeader(output, CODEC_NAME, VERSION_CURRENT, info.info.getId(), "");
- output.writeLong(gen);
+ CodecUtil.writeIndexHeader(output, CODEC_NAME, VERSION_CURRENT, info.info.getId(), Long.toString(gen, Character.MAX_RADIX));
for (int i = 0; i < data.length; i++) {
output.writeLong(data[i]);
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50NormsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50NormsConsumer.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50NormsConsumer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50NormsConsumer.java Fri Oct 24 02:43:41 2014
@@ -47,6 +47,7 @@ class Lucene50NormsConsumer extends Norm
static final byte CONST_COMPRESSED = 2;
static final byte UNCOMPRESSED = 3;
static final byte INDIRECT = 4;
+ static final byte PATCHED = 5;
static final int BLOCK_SIZE = 1 << 14;
// threshold for indirect encoding, computed as 1 - 1/log2(maxint)
@@ -61,10 +62,10 @@ class Lucene50NormsConsumer extends Norm
try {
String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
data = state.directory.createOutput(dataName, state.context);
- CodecUtil.writeSegmentHeader(data, dataCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.writeIndexHeader(data, dataCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
meta = state.directory.createOutput(metaName, state.context);
- CodecUtil.writeSegmentHeader(meta, metaCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.writeIndexHeader(meta, metaCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
success = true;
} finally {
if (!success) {
@@ -82,6 +83,11 @@ class Lucene50NormsConsumer extends Norm
@Override
public void addNormsField(FieldInfo field, Iterable<Number> values) throws IOException {
+ writeNormsField(field, values, 0);
+ }
+
+ private void writeNormsField(FieldInfo field, Iterable<Number> values, int level) throws IOException {
+ assert level <= 1; // we only "recurse" once in the indirect case
meta.writeVInt(field.number);
long minValue = Long.MAX_VALUE;
long maxValue = Long.MIN_VALUE;
@@ -89,16 +95,12 @@ class Lucene50NormsConsumer extends Norm
NormMap uniqueValues = new NormMap();
int count = 0;
- int missingCount = 0;
for (Number nv : values) {
if (nv == null) {
throw new IllegalStateException("illegal norms data for field " + field.name + ", got null for value: " + count);
}
final long v = nv.longValue();
- if (v == 0) {
- missingCount++;
- }
minValue = Math.min(minValue, v);
maxValue = Math.max(maxValue, v);
@@ -115,9 +117,15 @@ class Lucene50NormsConsumer extends Norm
if (uniqueValues != null && uniqueValues.size == 1) {
// 0 bpv
addConstant(minValue);
- } else if (count > 256 && missingCount > count * INDIRECT_THRESHOLD) {
- // sparse encoding
- addIndirect(field, values, count, missingCount);
+ } else if (level == 0 && count > 256 && uniqueValues != null && uniqueValues.maxFreq() > count * INDIRECT_THRESHOLD) {
+ long commonValue = uniqueValues.getDecodeTable()[uniqueValues.maxOrd()];
+ if (commonValue == 0) {
+ // if the common value is missing, don't waste RAM on a bitset, since we won't be searching those docs
+ addIndirect(field, values, count, uniqueValues);
+ } else {
+ // otherwise, write a sparse bitset, where 1 indicates 'uncommon value'.
+ addPatched(field, values, count, uniqueValues);
+ }
} else if (uniqueValues != null) {
// small number of unique values: this is the typical case:
FormatAndBits compression = fastestFormatAndBits(uniqueValues.size-1);
@@ -200,10 +208,65 @@ class Lucene50NormsConsumer extends Norm
writer.finish();
}
- private void addIndirect(FieldInfo field, final Iterable<Number> values, int count, int missingCount) throws IOException {
- meta.writeVInt(count - missingCount);
+ // encodes only uncommon values in a sparse bitset
+ // access is constant time, and the common case is predictable
+ // exceptions nest either to CONST (if there are only 2 values), or INDIRECT (if there are > 2 values)
+ private void addPatched(FieldInfo field, final Iterable<Number> values, int count, NormMap uniqueValues) throws IOException {
+ final long decodeTable[] = uniqueValues.getDecodeTable();
+ int commonCount = uniqueValues.maxFreq();
+ final long commonValue = decodeTable[uniqueValues.maxOrd()];
+
+ meta.writeVInt(count - commonCount);
+ meta.writeByte(PATCHED);
+ meta.writeLong(data.getFilePointer());
+
+ // write docs with value
+ writeDocsWithValue(values, commonValue);
+
+ // write exceptions: only two cases make sense
+ // bpv = 1 (folded into sparse bitset already)
+ // bpv > 1 (add indirect exception table)
+ meta.writeVInt(field.number);
+ if (uniqueValues.size == 2) {
+ // special case: implicit in bitset
+ int otherOrd = uniqueValues.maxOrd() == 0 ? 1 : 0;
+ addConstant(decodeTable[otherOrd]);
+ } else {
+ // exception table
+ addIndirect(field, values, count, uniqueValues);
+ }
+ }
+
+ // encodes values as sparse array: keys[] and values[]
+ // access is log(N) where N = keys.length (slow!)
+ // so this is only appropriate as an exception table for patched, or when common value is 0 (wont be accessed by searching)
+ private void addIndirect(FieldInfo field, final Iterable<Number> values, int count, NormMap uniqueValues) throws IOException {
+ int commonCount = uniqueValues.maxFreq();
+ final long commonValue = uniqueValues.getDecodeTable()[uniqueValues.maxOrd()];
+
+ meta.writeVInt(count - commonCount);
meta.writeByte(INDIRECT);
meta.writeLong(data.getFilePointer());
+
+ // write docs with value
+ writeDocsWithValue(values, commonValue);
+
+ // write actual values
+ writeNormsField(field, new Iterable<Number>() {
+ @Override
+ public Iterator<Number> iterator() {
+ return new FilterIterator<Number,Number>(values.iterator()) {
+ @Override
+ protected boolean predicateFunction(Number value) {
+ return value.longValue() != commonValue;
+ }
+ };
+ }
+ }, 1);
+ }
+
+ private void writeDocsWithValue(final Iterable<Number> values, long commonValue) throws IOException {
+ data.writeLong(commonValue);
data.writeVInt(PackedInts.VERSION_CURRENT);
data.writeVInt(BLOCK_SIZE);
@@ -212,25 +275,12 @@ class Lucene50NormsConsumer extends Norm
int doc = 0;
for (Number n : values) {
long v = n.longValue();
- if (v != 0) {
+ if (v != commonValue) {
writer.add(doc);
}
doc++;
}
writer.finish();
-
- // write actual values
- addNormsField(field, new Iterable<Number>() {
- @Override
- public Iterator<Number> iterator() {
- return new FilterIterator<Number,Number>(values.iterator()) {
- @Override
- protected boolean predicateFunction(Number value) {
- return value.longValue() != 0;
- }
- };
- }
- });
}
@Override
@@ -259,6 +309,7 @@ class Lucene50NormsConsumer extends Norm
static class NormMap {
// we use short: at most we will add 257 values to this map before its rejected as too big above.
final short[] singleByteRange = new short[256];
+ final int[] freqs = new int[257];
final Map<Long,Short> other = new HashMap<Long,Short>();
int size;
@@ -273,18 +324,24 @@ class Lucene50NormsConsumer extends Norm
int index = (int) (l + 128);
short previous = singleByteRange[index];
if (previous < 0) {
- singleByteRange[index] = (short) size;
+ short slot = (short) size;
+ singleByteRange[index] = slot;
+ freqs[slot]++;
size++;
return true;
} else {
+ freqs[previous]++;
return false;
}
} else {
- if (!other.containsKey(l)) {
+ Short previous = other.get(l);
+ if (previous == null) {
+ freqs[size]++;
other.put(l, (short)size);
size++;
return true;
} else {
+ freqs[previous]++;
return false;
}
}
@@ -315,5 +372,35 @@ class Lucene50NormsConsumer extends Norm
}
return decode;
}
+
+ // TODO: if we need more complicated frequency-driven optos, maybe add 'finish' to this api
+ // and sort all ords by frequency. we could then lower BPV and waste a value to represent 'patched',
+
+ /** retrieves frequency table for items (indexed by ordinal) */
+ public int[] getFreqs() {
+ return freqs;
+ }
+
+ /** sugar: returns max value over getFreqs() */
+ public int maxFreq() {
+ int max = 0;
+ for (int i = 0; i < size; i++) {
+ max = Math.max(max, freqs[i]);
+ }
+ return max;
+ }
+
+ /** sugar: returns ordinal with maxFreq() */
+ public int maxOrd() {
+ long max = 0;
+ int maxOrd = 0;
+ for (int i = 0; i < size; i++) {
+ if (freqs[i] > max) {
+ max = freqs[i];
+ maxOrd = i;
+ }
+ }
+ return maxOrd;
+ }
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50NormsFormat.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50NormsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50NormsFormat.java Fri Oct 24 02:43:41 2014
@@ -28,6 +28,7 @@ import org.apache.lucene.index.SegmentWr
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.SmallFloat;
import org.apache.lucene.util.packed.BlockPackedWriter;
+import org.apache.lucene.util.packed.MonotonicBlockPackedWriter;
import org.apache.lucene.util.packed.PackedInts;
/**
@@ -50,6 +51,9 @@ import org.apache.lucene.util.packed.Pac
* <li>Indirect: when norms are extremely sparse, missing values are omitted.
* Access to an individual value is slower, but missing norm values are never accessed
* by search code.
+ * <li>Patched: when a single norm value dominates, a sparse bitset encodes docs with exceptions,
+ * so that access to the common value is still very fast. outliers fall thru to an exception
+ * handling mechanism (Indirect or Constant).
* </ul>
* <p>
* Files:
@@ -64,7 +68,7 @@ import org.apache.lucene.util.packed.Pac
* Norms data (.nvd)</p>
* <p>Norms metadata (.dvm) --> Header,<Entry><sup>NumFields</sup>,Footer</p>
* <ul>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}</li>
* <li>Entry --> FieldNumber,Type,Offset</li>
* <li>FieldNumber --> {@link DataOutput#writeVInt vInt}</li>
* <li>Type --> {@link DataOutput#writeByte Byte}</li>
@@ -81,20 +85,24 @@ import org.apache.lucene.util.packed.Pac
* a lookup table of unique values is written, followed by the ordinal for each document.
* <li>2 --> constant. When there is a single value for the entire field.
* <li>3 --> uncompressed: Values written as a simple byte[].
- * <li>4 --> indirect. Only documents with a value are written with a sparse encoding.
+ * <li>4 --> indirect. Only documents with a value are written with monotonic compression. a nested
+ * entry for the same field will follow for the exception handler.
+ * <li>5 --> patched. Encoded the same as indirect.
* </ul>
* <li><a name="nvd" id="nvd"></a>
* <p>The Norms data or .nvd file.</p>
* <p>For each Norms field, this stores the actual per-document data (the heavy-lifting)</p>
- * <p>Norms data (.nvd) --> Header,<Uncompressed | TableCompressed | DeltaCompressed><sup>NumFields</sup>,Footer</p>
+ * <p>Norms data (.nvd) --> Header,<Uncompressed | TableCompressed | DeltaCompressed | MonotonicCompressed ><sup>NumFields</sup>,Footer</p>
* <ul>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}</li>
* <li>Uncompressed --> {@link DataOutput#writeByte Byte}<sup>maxDoc</sup></li>
* <li>TableCompressed --> PackedIntsVersion,Table,BitPackedData</li>
* <li>Table --> TableSize, {@link DataOutput#writeLong int64}<sup>TableSize</sup></li>
* <li>BitpackedData --> {@link PackedInts}</li>
* <li>DeltaCompressed --> PackedIntsVersion,BlockSize,DeltaCompressedData</li>
* <li>DeltaCompressedData --> {@link BlockPackedWriter BlockPackedWriter(blockSize=16k)}</li>
+ * <li>MonotonicCompressed --> PackedIntsVersion,BlockSize,MonotonicCompressedData</li>
+ * <li>MonotonicCompressedData --> {@link MonotonicBlockPackedWriter MonotonicBlockPackedWriter(blockSize=16k)}</li>
* <li>PackedIntsVersion,BlockSize,TableSize --> {@link DataOutput#writeVInt vInt}</li>
* <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50NormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50NormsProducer.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50NormsProducer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50NormsProducer.java Fri Oct 24 02:43:41 2014
@@ -37,6 +37,7 @@ import org.apache.lucene.util.Accountabl
import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.SparseFixedBitSet;
import org.apache.lucene.util.packed.BlockPackedReader;
import org.apache.lucene.util.packed.MonotonicBlockPackedReader;
import org.apache.lucene.util.packed.PackedInts;
@@ -48,6 +49,7 @@ import static org.apache.lucene.codecs.l
import static org.apache.lucene.codecs.lucene50.Lucene50NormsConsumer.TABLE_COMPRESSED;
import static org.apache.lucene.codecs.lucene50.Lucene50NormsConsumer.UNCOMPRESSED;
import static org.apache.lucene.codecs.lucene50.Lucene50NormsConsumer.INDIRECT;
+import static org.apache.lucene.codecs.lucene50.Lucene50NormsConsumer.PATCHED;
/**
* Reader for {@link Lucene50NormsFormat}
@@ -63,6 +65,7 @@ class Lucene50NormsProducer extends Norm
private final AtomicLong ramBytesUsed;
private final AtomicInteger activeCount = new AtomicInteger();
+ private final int maxDoc;
private final boolean merging;
@@ -75,11 +78,13 @@ class Lucene50NormsProducer extends Norm
instancesInfo.putAll(original.instancesInfo);
ramBytesUsed = new AtomicLong(original.ramBytesUsed.get());
activeCount.set(original.activeCount.get());
+ maxDoc = original.maxDoc;
merging = true;
}
Lucene50NormsProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
merging = false;
+ maxDoc = state.segmentInfo.getDocCount();
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
int version = -1;
@@ -88,7 +93,7 @@ class Lucene50NormsProducer extends Norm
try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context)) {
Throwable priorE = null;
try {
- version = CodecUtil.checkSegmentHeader(in, metaCodec, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+ version = CodecUtil.checkIndexHeader(in, metaCodec, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
readFields(in, state.fieldInfos);
} catch (Throwable exception) {
priorE = exception;
@@ -101,7 +106,7 @@ class Lucene50NormsProducer extends Norm
this.data = state.directory.openInput(dataName, state.context);
boolean success = false;
try {
- final int version2 = CodecUtil.checkSegmentHeader(data, dataCodec, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+ final int version2 = CodecUtil.checkIndexHeader(data, dataCodec, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
if (version != version2) {
throw new CorruptIndexException("Format versions mismatch: meta=" + version + ",data=" + version2, data);
}
@@ -146,6 +151,7 @@ class Lucene50NormsProducer extends Norm
case TABLE_COMPRESSED:
case DELTA_COMPRESSED:
break;
+ case PATCHED:
case INDIRECT:
if (meta.readVInt() != info.number) {
throw new CorruptIndexException("indirect norms entry for field: " + info.name + " is corrupt", meta);
@@ -254,6 +260,7 @@ class Lucene50NormsProducer extends Norm
}
case INDIRECT: {
data.seek(entry.offset);
+ final long common = data.readLong();
int packedIntsVersion = data.readVInt();
int blockSize = data.readVInt();
final MonotonicBlockPackedReader live = MonotonicBlockPackedReader.of(data, packedIntsVersion, blockSize, entry.count, false);
@@ -279,7 +286,34 @@ class Lucene50NormsProducer extends Norm
return values.get(mid);
}
}
- return 0;
+ return common;
+ }
+ };
+ break;
+ }
+ case PATCHED: {
+ data.seek(entry.offset);
+ final long common = data.readLong();
+ int packedIntsVersion = data.readVInt();
+ int blockSize = data.readVInt();
+ MonotonicBlockPackedReader live = MonotonicBlockPackedReader.of(data, packedIntsVersion, blockSize, entry.count, true);
+ final SparseFixedBitSet set = new SparseFixedBitSet(maxDoc);
+ for (int i = 0; i < live.size(); i++) {
+ int doc = (int) live.get(i);
+ set.set(doc);
+ }
+ LoadedNorms nestedInstance = loadNorms(entry.nested);
+ instance.ramBytesUsed = set.ramBytesUsed() + nestedInstance.ramBytesUsed;
+ instance.info = Accountables.namedAccountable("patched -> " + nestedInstance.info, instance.ramBytesUsed);
+ final NumericDocValues values = nestedInstance.norms;
+ instance.norms = new NumericDocValues() {
+ @Override
+ public long get(int docID) {
+ if (set.get(docID)) {
+ return values.get(docID);
+ } else {
+ return common;
+ }
}
};
break;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java Fri Oct 24 02:43:41 2014
@@ -34,7 +34,6 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.Version;
/**
@@ -42,20 +41,19 @@ import org.apache.lucene.util.Version;
* <p>
* Files:
* <ul>
- * <li><tt>.si</tt>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files, Id, Footer
+ * <li><tt>.si</tt>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files, Footer
* </ul>
* </p>
* Data types:
* <p>
* <ul>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}</li>
* <li>SegSize --> {@link DataOutput#writeInt Int32}</li>
* <li>SegVersion --> {@link DataOutput#writeString String}</li>
* <li>Files --> {@link DataOutput#writeStringSet Set<String>}</li>
* <li>Diagnostics --> {@link DataOutput#writeStringStringMap Map<String,String>}</li>
* <li>IsCompoundFile --> {@link DataOutput#writeByte Int8}</li>
* <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
- * <li>Id --> {@link DataOutput#writeString String}</li>
* </ul>
* </p>
* Field Descriptions:
@@ -84,15 +82,16 @@ public class Lucene50SegmentInfoFormat e
}
@Override
- public SegmentInfo read(Directory dir, String segment, IOContext context) throws IOException {
+ public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segment, "", Lucene50SegmentInfoFormat.SI_EXTENSION);
try (ChecksumIndexInput input = dir.openChecksumInput(fileName, context)) {
Throwable priorE = null;
SegmentInfo si = null;
try {
- CodecUtil.checkHeader(input, Lucene50SegmentInfoFormat.CODEC_NAME,
- Lucene50SegmentInfoFormat.VERSION_START,
- Lucene50SegmentInfoFormat.VERSION_CURRENT);
+ CodecUtil.checkIndexHeader(input, Lucene50SegmentInfoFormat.CODEC_NAME,
+ Lucene50SegmentInfoFormat.VERSION_START,
+ Lucene50SegmentInfoFormat.VERSION_CURRENT,
+ segmentID, "");
final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt());
final int docCount = input.readInt();
@@ -103,10 +102,7 @@ public class Lucene50SegmentInfoFormat e
final Map<String,String> diagnostics = input.readStringStringMap();
final Set<String> files = input.readStringSet();
- byte[] id = new byte[StringHelper.ID_LENGTH];
- input.readBytes(id, 0, id.length);
-
- si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, id);
+ si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, segmentID);
si.setFiles(files);
} catch (Throwable exception) {
priorE = exception;
@@ -124,7 +120,11 @@ public class Lucene50SegmentInfoFormat e
boolean success = false;
try (IndexOutput output = dir.createOutput(fileName, ioContext)) {
- CodecUtil.writeHeader(output, Lucene50SegmentInfoFormat.CODEC_NAME, Lucene50SegmentInfoFormat.VERSION_CURRENT);
+ CodecUtil.writeIndexHeader(output,
+ Lucene50SegmentInfoFormat.CODEC_NAME,
+ Lucene50SegmentInfoFormat.VERSION_CURRENT,
+ si.getId(),
+ "");
Version version = si.getVersion();
if (version.major < 5) {
throw new IllegalArgumentException("invalid major version: should be >= 5 but got: " + version.major + " segment=" + si);
@@ -145,11 +145,6 @@ public class Lucene50SegmentInfoFormat e
}
}
output.writeStringSet(files);
- byte[] id = si.getId();
- if (id.length != StringHelper.ID_LENGTH) {
- throw new IllegalArgumentException("invalid id, got=" + StringHelper.idToString(id));
- }
- output.writeBytes(id, 0, id.length);
CodecUtil.writeFooter(output);
success = true;
} finally {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java Fri Oct 24 02:43:41 2014
@@ -52,7 +52,7 @@ import org.apache.lucene.util.packed.Pac
* <p>Here is a more detailed description of the field data file format:</p>
* <ul>
* <li>FieldData (.fdt) --> <Header>, PackedIntsVersion, <Chunk><sup>ChunkCount</sup></li>
- * <li>Header --> {@link CodecUtil#writeSegmentHeader SegmentHeader}</li>
+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}</li>
* <li>PackedIntsVersion --> {@link PackedInts#VERSION_CURRENT} as a {@link DataOutput#writeVInt VInt}</li>
* <li>ChunkCount is not known in advance and is the number of chunks necessary to store all document of the segment</li>
* <li>Chunk --> DocBase, ChunkDocs, DocFieldCounts, DocLengths, <CompressedDocs></li>
@@ -104,7 +104,7 @@ import org.apache.lucene.util.packed.Pac
* <p>A fields index file (extension <tt>.fdx</tt>).</p>
* <ul>
* <li>FieldsIndex (.fdx) --> <Header>, <ChunkIndex></li>
- * <li>Header --> {@link CodecUtil#writeSegmentHeader SegmentHeader}</li>
+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}</li>
* <li>ChunkIndex: See {@link CompressingStoredFieldsIndexWriter}</li>
* </ul>
* </li>
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.java Fri Oct 24 02:43:41 2014
@@ -59,7 +59,7 @@ import org.apache.lucene.util.packed.Pac
* <p>Here is a more detailed description of the field data file format:</p>
* <ul>
* <li>VectorData (.tvd) --> <Header>, PackedIntsVersion, ChunkSize, <Chunk><sup>ChunkCount</sup>, Footer</li>
- * <li>Header --> {@link CodecUtil#writeSegmentHeader SegmentHeader}</li>
+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}</li>
* <li>PackedIntsVersion --> {@link PackedInts#VERSION_CURRENT} as a {@link DataOutput#writeVInt VInt}</li>
* <li>ChunkSize is the number of bytes of terms to accumulate before flushing, as a {@link DataOutput#writeVInt VInt}</li>
* <li>ChunkCount is not known in advance and is the number of chunks necessary to store all document of the segment</li>
@@ -113,7 +113,7 @@ import org.apache.lucene.util.packed.Pac
* <p>An index file (extension <tt>.tvx</tt>).</p>
* <ul>
* <li>VectorIndex (.tvx) --> <Header>, <ChunkIndex>, Footer</li>
- * <li>Header --> {@link CodecUtil#writeSegmentHeader SegmentHeader}</li>
+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}</li>
* <li>ChunkIndex: See {@link CompressingStoredFieldsIndexWriter}</li>
* <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/package.html?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/package.html (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/package.html Fri Oct 24 02:43:41 2014
@@ -154,20 +154,20 @@ its title, url, or an identifier to acce
returned for each hit when searching. This is keyed by document number.
</li>
<li>
-{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term dictionary}.
+{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Term dictionary}.
A dictionary containing all of the terms used in all of the
indexed fields of all of the documents. The dictionary also contains the number
of documents which contain the term, and pointers to the term's frequency and
proximity data.
</li>
<li>
-{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Frequency data}.
+{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Term Frequency data}.
For each term in the dictionary, the numbers of all the
documents that contain that term, and the frequency of the term in that
document, unless frequencies are omitted (IndexOptions.DOCS_ONLY)
</li>
<li>
-{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Proximity data}.
+{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Term Proximity data}.
For each term in the dictionary, the positions that the
term occurs in each document. Note that this will not exist if all fields in
all documents omit position data.
@@ -185,7 +185,7 @@ term frequency. To add Term Vectors to y
{@link org.apache.lucene.document.Field Field} constructors
</li>
<li>
-{@link org.apache.lucene.codecs.lucene410.Lucene410DocValuesFormat Per-document values}.
+{@link org.apache.lucene.codecs.lucene50.Lucene50DocValuesFormat Per-document values}.
Like stored values, these are also keyed by document
number, but are generally intended to be loaded into main memory for fast
access. Whereas stored values are generally intended for summary results from
@@ -264,27 +264,27 @@ systems that frequently run out of file
<td>The stored fields for documents</td>
</tr>
<tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Dictionary}</td>
+<td>{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Term Dictionary}</td>
<td>.tim</td>
<td>The term dictionary, stores term info</td>
</tr>
<tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Index}</td>
+<td>{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Term Index}</td>
<td>.tip</td>
<td>The index into the Term Dictionary</td>
</tr>
<tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Frequencies}</td>
+<td>{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Frequencies}</td>
<td>.doc</td>
<td>Contains the list of docs which contain each term along with frequency</td>
</tr>
<tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Positions}</td>
+<td>{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Positions}</td>
<td>.pos</td>
<td>Stores position information about where a term occurs in the index</td>
</tr>
<tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Payloads}</td>
+<td>{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Payloads}</td>
<td>.pay</td>
<td>Stores additional per-position metadata information such as character offsets and user payloads</td>
</tr>
@@ -294,7 +294,7 @@ systems that frequently run out of file
<td>Encodes length and boost factors for docs and fields</td>
</tr>
<tr>
-<td>{@link org.apache.lucene.codecs.lucene410.Lucene410DocValuesFormat Per-Document Values}</td>
+<td>{@link org.apache.lucene.codecs.lucene50.Lucene50DocValuesFormat Per-Document Values}</td>
<td>.dvd, .dvm</td>
<td>Encodes additional scoring factors or other per-document information.</td>
</tr>
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Fri Oct 24 02:43:41 2014
@@ -32,8 +32,6 @@ import java.util.Map;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.blocktree.FieldReader;
-import org.apache.lucene.codecs.blocktree.Stats;
import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.DocIdSetIterator;
@@ -278,7 +276,7 @@ public class CheckIndex implements Close
* tree terms dictionary (this is only set if the
* {@link PostingsFormat} for this segment uses block
* tree. */
- public Map<String,Stats> blockTreeStats = null;
+ public Map<String,Object> blockTreeStats = null;
}
/**
@@ -453,7 +451,7 @@ public class CheckIndex implements Close
public Status checkIndex(List<String> onlySegments) throws IOException {
ensureOpen();
NumberFormat nf = NumberFormat.getInstance(Locale.ROOT);
- SegmentInfos sis = new SegmentInfos();
+ SegmentInfos sis = null;
Status result = new Status();
result.dir = dir;
String[] files = dir.listAll();
@@ -464,7 +462,7 @@ public class CheckIndex implements Close
try {
// Do not use SegmentInfos.read(Directory) since the spooky
// retrying it does is not necessary here (we hold the write lock):
- sis.read(dir, lastSegmentsFile);
+ sis = SegmentInfos.readCommit(dir, lastSegmentsFile);
} catch (Throwable t) {
if (failFast) {
IOUtils.reThrow(t);
@@ -1289,14 +1287,12 @@ public class CheckIndex implements Close
// docs got deleted and then merged away):
} else {
- if (fieldTerms instanceof FieldReader) {
- final Stats stats = ((FieldReader) fieldTerms).computeStats();
- assert stats != null;
- if (status.blockTreeStats == null) {
- status.blockTreeStats = new HashMap<>();
- }
- status.blockTreeStats.put(field, stats);
+ final Object stats = fieldTerms.getStats();
+ assert stats != null;
+ if (status.blockTreeStats == null) {
+ status.blockTreeStats = new HashMap<>();
}
+ status.blockTreeStats.put(field, stats);
if (sumTotalTermFreq != 0) {
final long v = fields.terms(field).getSumTotalTermFreq();
@@ -1423,7 +1419,7 @@ public class CheckIndex implements Close
}
if (verbose && status.blockTreeStats != null && infoStream != null && status.termCount > 0) {
- for(Map.Entry<String,Stats> ent : status.blockTreeStats.entrySet()) {
+ for(Map.Entry<String, Object> ent : status.blockTreeStats.entrySet()) {
infoStream.println(" field \"" + ent.getKey() + "\":");
infoStream.println(" " + ent.getValue().toString().replace("\n", "\n "));
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java Fri Oct 24 02:43:41 2014
@@ -225,8 +225,7 @@ public abstract class DirectoryReader ex
List<IndexCommit> commits = new ArrayList<>();
- SegmentInfos latest = new SegmentInfos();
- latest.read(dir);
+ SegmentInfos latest = SegmentInfos.readLatestCommit(dir);
final long currentGen = latest.getGeneration();
commits.add(new StandardDirectoryReader.ReaderCommit(latest, dir));
@@ -239,11 +238,11 @@ public abstract class DirectoryReader ex
!fileName.equals(IndexFileNames.OLD_SEGMENTS_GEN) &&
SegmentInfos.generationFromSegmentsFileName(fileName) < currentGen) {
- SegmentInfos sis = new SegmentInfos();
+ SegmentInfos sis = null;
try {
// IOException allowed to throw there, in case
// segments_N is corrupt
- sis.read(dir, fileName);
+ sis = SegmentInfos.readCommit(dir, fileName);
} catch (FileNotFoundException | NoSuchFileException fnfe) {
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
@@ -252,7 +251,6 @@ public abstract class DirectoryReader ex
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
- sis = null;
}
if (sis != null)
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java Fri Oct 24 02:43:41 2014
@@ -151,6 +151,11 @@ public class FilterLeafReader extends Le
public boolean hasPayloads() {
return in.hasPayloads();
}
+
+ @Override
+ public Object getStats() throws IOException {
+ return in.getStats();
+ }
}
/** Base class for filtering {@link TermsEnum} implementations. */
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java Fri Oct 24 02:43:41 2014
@@ -165,9 +165,9 @@ final class IndexFileDeleter implements
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "init: load commit \"" + fileName + "\"");
}
- SegmentInfos sis = new SegmentInfos();
+ SegmentInfos sis = null;
try {
- sis.read(directory, fileName);
+ sis = SegmentInfos.readCommit(directory, fileName);
} catch (FileNotFoundException | NoSuchFileException e) {
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
@@ -179,7 +179,6 @@ final class IndexFileDeleter implements
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
}
- sis = null;
} catch (IOException e) {
if (SegmentInfos.generationFromSegmentsFileName(fileName) <= currentGen && directory.fileLength(fileName) > 0) {
throw e;
@@ -187,7 +186,6 @@ final class IndexFileDeleter implements
// Most likely we are opening an index that
// has an aborted "future" commit, so suppress
// exc in this case
- sis = null;
}
}
if (sis != null) {
@@ -215,9 +213,9 @@ final class IndexFileDeleter implements
// listing was stale (eg when index accessed via NFS
// client with stale directory listing cache). So we
// try now to explicitly open this commit point:
- SegmentInfos sis = new SegmentInfos();
+ SegmentInfos sis = null;
try {
- sis.read(directory, currentSegmentsFile);
+ sis = SegmentInfos.readCommit(directory, currentSegmentsFile);
} catch (IOException e) {
throw new CorruptIndexException("unable to read current segments_N file", currentSegmentsFile, e);
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Fri Oct 24 02:43:41 2014
@@ -773,7 +773,6 @@ public class IndexWriter implements Clos
// If index is too old, reading the segments will throw
// IndexFormatTooOldException.
- segmentInfos = new SegmentInfos();
boolean initialIndexExists = true;
@@ -782,13 +781,17 @@ public class IndexWriter implements Clos
// against an index that's currently open for
// searching. In this case we write the next
// segments_N file with no segments:
+ SegmentInfos sis = null;
try {
- segmentInfos.read(directory);
- segmentInfos.clear();
+ sis = SegmentInfos.readLatestCommit(directory);
+ sis.clear();
} catch (IOException e) {
// Likely this means it's a fresh directory
initialIndexExists = false;
+ sis = new SegmentInfos();
}
+
+ segmentInfos = sis;
// Record that we have a change (zero out all
// segments) pending:
@@ -802,7 +805,7 @@ public class IndexWriter implements Clos
// Do not use SegmentInfos.read(Directory) since the spooky
// retrying it does is not necessary here (we hold the write lock):
- segmentInfos.read(directory, lastSegmentsFile);
+ segmentInfos = SegmentInfos.readCommit(directory, lastSegmentsFile);
IndexCommit commit = config.getIndexCommit();
if (commit != null) {
@@ -813,8 +816,7 @@ public class IndexWriter implements Clos
// points.
if (commit.getDirectory() != directory)
throw new IllegalArgumentException("IndexCommit's directory doesn't match my directory");
- SegmentInfos oldInfos = new SegmentInfos();
- oldInfos.read(directory, commit.getSegmentsFileName());
+ SegmentInfos oldInfos = SegmentInfos.readCommit(directory, commit.getSegmentsFileName());
segmentInfos.replace(oldInfos);
changed();
if (infoStream.isEnabled("IW")) {
@@ -2401,8 +2403,7 @@ public class IndexWriter implements Clos
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "addIndexes: process directory " + dir);
}
- SegmentInfos sis = new SegmentInfos(); // read infos from dir
- sis.read(dir);
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir); // read infos from dir
totalDocCount += sis.totalDocCount();
for (SegmentCommitInfo info : sis) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java Fri Oct 24 02:43:41 2014
@@ -60,18 +60,20 @@ import org.apache.lucene.util.StringHelp
* Files:
* <ul>
* <li><tt>segments_N</tt>: Header, Version, NameCounter, SegCount, <SegName,
- * SegCodec, DelGen, DeletionCount, FieldInfosGen, DocValuesGen,
+ * HasSegID, SegID, SegCodec, DelGen, DeletionCount, FieldInfosGen, DocValuesGen,
* UpdatesFiles><sup>SegCount</sup>, CommitUserData, Footer
* </ul>
* </p>
* Data types:
* <p>
* <ul>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>GenHeader, NameCounter, SegCount, DeletionCount -->
+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}</li>
+ * <li>NameCounter, SegCount, DeletionCount -->
* {@link DataOutput#writeInt Int32}</li>
* <li>Generation, Version, DelGen, Checksum, FieldInfosGen, DocValuesGen -->
* {@link DataOutput#writeLong Int64}</li>
+ * <li>HasSegID --> {@link DataOutput#writeByte Int8}</li>
+ * <li>SegID --> {@link DataOutput#writeByte Int8<sup>ID_LENGTH</sup>}</li>
* <li>SegName, SegCodec --> {@link DataOutput#writeString String}</li>
* <li>CommitUserData --> {@link DataOutput#writeStringStringMap
* Map<String,String>}</li>
@@ -94,6 +96,10 @@ import org.apache.lucene.util.StringHelp
* <li>DeletionCount records the number of deleted documents in this segment.</li>
* <li>SegCodec is the {@link Codec#getName() name} of the Codec that encoded
* this segment.</li>
+ * <li>HasSegID is nonzero if the segment has an identifier. Otherwise, when it is 0
+ * the identifier is {@code null} and no SegID is written. Null only happens for Lucene
+ * 4.x segments referenced in commits.</li>
+ * <li>SegID is the identifier of the Codec that encoded this segment. </li>
* <li>CommitUserData stores an optional user-supplied opaque
* Map<String,String> that was passed to
* {@link IndexWriter#setCommitData(java.util.Map)}.</li>
@@ -155,8 +161,8 @@ public final class SegmentInfos implemen
private byte[] id;
/** Sole constructor. Typically you call this and then
- * use {@link #read(Directory) or
- * #read(Directory,String)} to populate each {@link
+ * use {@link #readLatestCommit(Directory) or
+ * #readCommit(Directory,String)} to populate each {@link
* SegmentCommitInfo}. Alternatively, you can add/remove your
* own {@link SegmentCommitInfo}s. */
public SegmentInfos() {
@@ -245,21 +251,14 @@ public final class SegmentInfos implemen
throw new IllegalArgumentException("fileName \"" + fileName + "\" is not a segments file");
}
}
-
- /**
- * Get the next pending_segments_N filename that will be written.
- */
- public String getNextPendingSegmentFileName() {
- long nextGeneration;
-
+
+ /** return generation of the next pending_segments_N that will be written */
+ private long getNextPendingGeneration() {
if (generation == -1) {
- nextGeneration = 1;
+ return 1;
} else {
- nextGeneration = generation+1;
+ return generation+1;
}
- return IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS,
- "",
- nextGeneration);
}
/** Since Lucene 5.0, every commit (segments_N) writes a unique id. This will
@@ -277,18 +276,10 @@ public final class SegmentInfos implemen
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public final void read(Directory directory, String segmentFileName) throws IOException {
- boolean success = false;
-
- // Clear any previous segments:
- this.clear();
-
- generation = generationFromSegmentsFileName(segmentFileName);
+ public static final SegmentInfos readCommit(Directory directory, String segmentFileName) throws IOException {
- lastGeneration = generation;
-
- ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READ);
- try {
+ long generation = generationFromSegmentsFileName(segmentFileName);
+ try (ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READ)) {
// NOTE: as long as we want to throw indexformattooold (vs corruptindexexception), we need
// to read the magic ourselves.
int magic = input.readInt();
@@ -297,17 +288,42 @@ public final class SegmentInfos implemen
}
// 4.0+
int format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_40, VERSION_50);
- version = input.readLong();
- counter = input.readInt();
+ // 5.0+
+ byte id[] = null;
+ if (format >= VERSION_50) {
+ id = new byte[StringHelper.ID_LENGTH];
+ input.readBytes(id, 0, id.length);
+ CodecUtil.checkIndexHeaderSuffix(input, Long.toString(generation, Character.MAX_RADIX));
+ }
+
+ SegmentInfos infos = new SegmentInfos();
+ infos.id = id;
+ infos.generation = generation;
+ infos.lastGeneration = generation;
+ infos.version = input.readLong();
+ infos.counter = input.readInt();
int numSegments = input.readInt();
if (numSegments < 0) {
throw new CorruptIndexException("invalid segment count: " + numSegments, input);
}
for (int seg = 0; seg < numSegments; seg++) {
String segName = input.readString();
+ final byte segmentID[];
+ if (format >= VERSION_50) {
+ byte hasID = input.readByte();
+ if (hasID == 1) {
+ segmentID = new byte[StringHelper.ID_LENGTH];
+ input.readBytes(segmentID, 0, segmentID.length);
+ } else if (hasID == 0) {
+ segmentID = null; // 4.x segment, doesn't have an ID
+ } else {
+ throw new CorruptIndexException("invalid hasID byte, got: " + hasID, input);
+ }
+ } else {
+ segmentID = null;
+ }
Codec codec = Codec.forName(input.readString());
- //System.out.println("SIS.read seg=" + seg + " codec=" + codec);
- SegmentInfo info = codec.segmentInfoFormat().read(directory, segName, IOContext.READ);
+ SegmentInfo info = codec.segmentInfoFormat().read(directory, segName, segmentID, IOContext.READ);
info.setCodec(codec);
long delGen = input.readLong();
int delCount = input.readInt();
@@ -358,13 +374,9 @@ public final class SegmentInfos implemen
siPerCommit.setDocValuesUpdatesFiles(dvUpdateFiles);
}
}
- add(siPerCommit);
- }
- userData = input.readStringStringMap();
- if (format >= VERSION_50) {
- id = new byte[StringHelper.ID_LENGTH];
- input.readBytes(id, 0, id.length);
+ infos.add(siPerCommit);
}
+ infos.userData = input.readStringStringMap();
if (format >= VERSION_48) {
CodecUtil.checkFooter(input);
@@ -378,30 +390,17 @@ public final class SegmentInfos implemen
CodecUtil.checkEOF(input);
}
- success = true;
- } finally {
- if (!success) {
- // Clear any segment infos we had loaded so we
- // have a clean slate on retry:
- this.clear();
- IOUtils.closeWhileHandlingException(input);
- } else {
- input.close();
- }
+ return infos;
}
}
/** Find the latest commit ({@code segments_N file}) and
* load all {@link SegmentCommitInfo}s. */
- public final void read(Directory directory) throws IOException {
- generation = lastGeneration = -1;
-
- new FindSegmentsFile(directory) {
-
+ public static final SegmentInfos readLatestCommit(Directory directory) throws IOException {
+ return new FindSegmentsFile<SegmentInfos>(directory) {
@Override
- protected Object doBody(String segmentFileName) throws IOException {
- read(directory, segmentFileName);
- return null;
+ protected SegmentInfos doBody(String segmentFileName) throws IOException {
+ return readCommit(directory, segmentFileName);
}
}.run();
}
@@ -412,27 +411,38 @@ public final class SegmentInfos implemen
private void write(Directory directory) throws IOException {
- String segmentFileName = getNextPendingSegmentFileName();
+ long nextGeneration = getNextPendingGeneration();
+ String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS,
+ "",
+ nextGeneration);
// Always advance the generation on write:
- if (generation == -1) {
- generation = 1;
- } else {
- generation++;
- }
+ generation = nextGeneration;
IndexOutput segnOutput = null;
boolean success = false;
try {
segnOutput = directory.createOutput(segmentFileName, IOContext.DEFAULT);
- CodecUtil.writeHeader(segnOutput, "segments", VERSION_50);
+ CodecUtil.writeIndexHeader(segnOutput, "segments", VERSION_50,
+ StringHelper.randomId(), Long.toString(nextGeneration, Character.MAX_RADIX));
segnOutput.writeLong(version);
segnOutput.writeInt(counter); // write counter
segnOutput.writeInt(size()); // write infos
for (SegmentCommitInfo siPerCommit : this) {
SegmentInfo si = siPerCommit.info;
segnOutput.writeString(si.name);
+ byte segmentID[] = si.getId();
+ // TODO: remove this in lucene 6, we don't need to include 4.x segments in commits anymore
+ if (segmentID == null) {
+ segnOutput.writeByte((byte)0);
+ } else {
+ if (segmentID.length != StringHelper.ID_LENGTH) {
+ throw new IllegalStateException("cannot write segment: invalid id segment=" + si.name + "id=" + StringHelper.idToString(segmentID));
+ }
+ segnOutput.writeByte((byte)1);
+ segnOutput.writeBytes(segmentID, segmentID.length);
+ }
segnOutput.writeString(si.getCodec().getName());
segnOutput.writeLong(siPerCommit.getDelGen());
int delCount = siPerCommit.getDelCount();
@@ -452,8 +462,6 @@ public final class SegmentInfos implemen
assert si.dir == directory;
}
segnOutput.writeStringStringMap(userData);
- byte[] id = StringHelper.randomId();
- segnOutput.writeBytes(id, 0, id.length);
CodecUtil.writeFooter(segnOutput);
segnOutput.close();
directory.sync(Collections.singleton(segmentFileName));
@@ -547,7 +555,7 @@ public final class SegmentInfos implemen
* time, etc., it could have been deleted due to a writer
* commit finishing.
*/
- public abstract static class FindSegmentsFile {
+ public abstract static class FindSegmentsFile<T> {
final Directory directory;
@@ -558,12 +566,12 @@ public final class SegmentInfos implemen
/** Locate the most recent {@code segments} file and
* run {@link #doBody} on it. */
- public Object run() throws IOException {
+ public T run() throws IOException {
return run(null);
}
/** Run {@link #doBody} on the provided commit. */
- public Object run(IndexCommit commit) throws IOException {
+ public T run(IndexCommit commit) throws IOException {
if (commit != null) {
if (directory != commit.getDirectory())
throw new IOException("the specified commit does not match the specified Directory");
@@ -607,11 +615,11 @@ public final class SegmentInfos implemen
String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
try {
- Object v = doBody(segmentFileName);
+ T t = doBody(segmentFileName);
if (infoStream != null) {
message("success on " + segmentFileName);
}
- return v;
+ return t;
} catch (IOException err) {
// Save the original root cause:
if (exc == null) {
@@ -634,7 +642,7 @@ public final class SegmentInfos implemen
* during the processing that could have been caused by
* a writer committing.
*/
- protected abstract Object doBody(String segmentFileName) throws IOException;
+ protected abstract T doBody(String segmentFileName) throws IOException;
}
// Carry over generation numbers from another SegmentInfos
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java Fri Oct 24 02:43:41 2014
@@ -47,11 +47,10 @@ final class StandardDirectoryReader exte
/** called from DirectoryReader.open(...) methods */
static DirectoryReader open(final Directory directory, final IndexCommit commit) throws IOException {
- return (DirectoryReader) new SegmentInfos.FindSegmentsFile(directory) {
+ return new SegmentInfos.FindSegmentsFile<DirectoryReader>(directory) {
@Override
- protected Object doBody(String segmentFileName) throws IOException {
- SegmentInfos sis = new SegmentInfos();
- sis.read(directory, segmentFileName);
+ protected DirectoryReader doBody(String segmentFileName) throws IOException {
+ SegmentInfos sis = SegmentInfos.readCommit(directory, segmentFileName);
final SegmentReader[] readers = new SegmentReader[sis.size()];
for (int i = sis.size()-1; i >= 0; i--) {
boolean success = false;
@@ -309,11 +308,10 @@ final class StandardDirectoryReader exte
}
private DirectoryReader doOpenFromCommit(IndexCommit commit) throws IOException {
- return (DirectoryReader) new SegmentInfos.FindSegmentsFile(directory) {
+ return new SegmentInfos.FindSegmentsFile<DirectoryReader>(directory) {
@Override
- protected Object doBody(String segmentFileName) throws IOException {
- final SegmentInfos infos = new SegmentInfos();
- infos.read(directory, segmentFileName);
+ protected DirectoryReader doBody(String segmentFileName) throws IOException {
+ final SegmentInfos infos = SegmentInfos.readCommit(directory, segmentFileName);
return doOpenIfChanged(infos);
}
}.run(commit);
@@ -338,8 +336,7 @@ final class StandardDirectoryReader exte
// IndexWriter.prepareCommit has been called (but not
// yet commit), then the reader will still see itself as
// current:
- SegmentInfos sis = new SegmentInfos();
- sis.read(directory);
+ SegmentInfos sis = SegmentInfos.readLatestCommit(directory);
// we loaded SegmentInfos from the directory
return sis.getVersion() == segmentInfos.getVersion();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/Terms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/Terms.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/Terms.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/Terms.java Fri Oct 24 02:43:41 2014
@@ -193,4 +193,18 @@ public abstract class Terms {
scratch.grow(scratch.length());
}
}
+
+ /**
+ * Expert: returns additional information about this Terms instance
+ * for debugging purposes.
+ */
+ public Object getStats() throws IOException {
+ StringBuilder sb = new StringBuilder();
+ sb.append("impl=" + getClass().getSimpleName());
+ sb.append(",size=" + size());
+ sb.append(",docCount=" + getDocCount());
+ sb.append(",sumTotalTermFreq=" + getSumTotalTermFreq());
+ sb.append(",sumDocFreq=" + getSumDocFreq());
+ return sb.toString();
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat (original)
+++ lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat Fri Oct 24 02:43:41 2014
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.lucene410.Lucene410DocValuesFormat
+org.apache.lucene.codecs.lucene50.Lucene50DocValuesFormat
Modified: lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original)
+++ lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Fri Oct 24 02:43:41 2014
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat
+org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50NormsFormat.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50NormsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50NormsFormat.java Fri Oct 24 02:43:41 2014
@@ -17,7 +17,13 @@ package org.apache.lucene.codecs.lucene5
* limitations under the License.
*/
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene50.Lucene50NormsConsumer.NormMap;
import org.apache.lucene.index.BaseNormsFormatTestCase;
import org.apache.lucene.util.TestUtil;
@@ -30,5 +36,92 @@ public class TestLucene50NormsFormat ext
@Override
protected Codec getCodec() {
return codec;
- }
+ }
+
+ // NormMap is rather complicated, doing domain encoding / tracking frequencies etc.
+ // test it directly some here...
+
+ public void testNormMapSimple() {
+ NormMap map = new NormMap();
+ map.add(10);
+ map.add(5);
+ map.add(4);
+ map.add(10);
+ assertEquals(3, map.size);
+
+ // first come, first serve ord assignment
+
+ // encode
+ assertEquals(0, map.getOrd(10));
+ assertEquals(1, map.getOrd(5));
+ assertEquals(2, map.getOrd(4));
+
+ // decode
+ long decode[] = map.getDecodeTable();
+ assertEquals(10, decode[0]);
+ assertEquals(5, decode[1]);
+ assertEquals(4, decode[2]);
+
+ // freqs
+ int freqs[] = map.getFreqs();
+ assertEquals(2, freqs[0]);
+ assertEquals(1, freqs[1]);
+ assertEquals(1, freqs[2]);
+
+ assertEquals(2, map.maxFreq());
+ }
+
+ public void testNormMapRandom() {
+ Map<Long,Integer> freqs = new HashMap<>();
+ Map<Long,Integer> ords = new HashMap<>();
+
+ Set<Long> uniqueValuesSet = new HashSet<>();
+ int numUniqValues = TestUtil.nextInt(random(), 1, 256);
+ for (int i = 0; i < numUniqValues; i++) {
+ if (random().nextBoolean()) {
+ uniqueValuesSet.add(TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE));
+ } else {
+ uniqueValuesSet.add(TestUtil.nextLong(random(), Byte.MIN_VALUE, Byte.MAX_VALUE));
+ }
+ }
+
+ Long uniqueValues[] = uniqueValuesSet.toArray(new Long[uniqueValuesSet.size()]);
+
+ NormMap map = new NormMap();
+ int numdocs = TestUtil.nextInt(random(), 1, 100000);
+ for (int i = 0; i < numdocs; i++) {
+ long value = uniqueValues[random().nextInt(uniqueValues.length)];
+ // now add to both expected and actual
+ map.add(value);
+
+ Integer ord = ords.get(value);
+ if (ord == null) {
+ ord = ords.size();
+ ords.put(value, ord);
+ freqs.put(value, 1);
+ } else {
+ freqs.put(value, freqs.get(value)+1);
+ }
+ }
+
+ // value -> ord
+ assertEquals(ords.size(), map.size);
+ for (Map.Entry<Long,Integer> kv : ords.entrySet()) {
+ assertEquals(kv.getValue().intValue(), map.getOrd(kv.getKey()));
+ }
+
+ // ord -> value
+ Map<Long,Integer> reversed = new HashMap<>();
+ long table[] = map.getDecodeTable();
+ for (int i = 0; i < map.size; i++) {
+ reversed.put(table[i], i);
+ }
+ assertEquals(ords, reversed);
+
+ // freqs
+ int freqTable[] = map.getFreqs();
+ for (int i = 0; i < map.size; i++) {
+ assertEquals(freqs.get(table[i]).longValue(), freqTable[i]);
+ }
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java Fri Oct 24 02:43:41 2014
@@ -22,7 +22,7 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.asserting.AssertingCodec;
-import org.apache.lucene.codecs.lucene41vargap.Lucene41VarGapFixedInterval;
+import org.apache.lucene.codecs.blockterms.LuceneVarGapFixedInterval;
import org.apache.lucene.codecs.memory.MemoryPostingsFormat;
import org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat;
import org.apache.lucene.document.Document;
@@ -287,9 +287,9 @@ public class TestPerFieldPostingsFormat2
@Override
public PostingsFormat getPostingsFormatForField(String field) {
if ("id".equals(field)) {
- return new Lucene41VarGapFixedInterval(1);
+ return new LuceneVarGapFixedInterval(1);
} else if ("date".equals(field)) {
- return new Lucene41VarGapFixedInterval(2);
+ return new LuceneVarGapFixedInterval(2);
} else {
return super.getPostingsFormatForField(field);
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java Fri Oct 24 02:43:41 2014
@@ -1116,8 +1116,7 @@ public class TestAddIndexes extends Luce
w3.close();
// we should now see segments_X,
// _Y.cfs,_Y.cfe, _Z.si
- SegmentInfos sis = new SegmentInfos();
- sis.read(dir);
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
assertEquals("Only one compound segment should exist", 1, sis.size());
assertTrue(sis.info(0).info.getUseCompoundFile());
dir.close();
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java Fri Oct 24 02:43:41 2014
@@ -26,7 +26,6 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -36,10 +35,6 @@ import org.apache.lucene.util.TestUtil;
public class TestAllFilesHaveChecksumFooter extends LuceneTestCase {
public void test() throws Exception {
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // Else we might remove .cfe but not the corresponding .cfs, causing false exc when trying to verify headers:
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
conf.setCodec(TestUtil.getDefaultCodec());
RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
@@ -68,8 +63,7 @@ public class TestAllFilesHaveChecksumFoo
}
private void checkFooters(Directory dir) throws IOException {
- SegmentInfos sis = new SegmentInfos();
- sis.read(dir);
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
checkFooter(dir, sis.getSegmentsFileName());
for (SegmentCommitInfo si : sis) {
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java Fri Oct 24 02:43:41 2014
@@ -30,22 +30,16 @@ import org.apache.lucene.document.Numeri
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
/**
- * Test that a plain default puts codec headers in all files.
+ * Test that a plain default puts codec headers in all files
*/
public class TestAllFilesHaveCodecHeader extends LuceneTestCase {
public void test() throws Exception {
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // Else we might remove .cfe but not the corresponding .cfs, causing false exc when trying to verify headers:
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
-
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
conf.setCodec(TestUtil.getDefaultCodec());
RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
@@ -70,10 +64,12 @@ public class TestAllFilesHaveCodecHeader
if (random().nextInt(7) == 0) {
riw.commit();
}
- // TODO: we should make a new format with a clean header...
- // if (random().nextInt(20) == 0) {
- // riw.deleteDocuments(new Term("id", Integer.toString(i)));
- // }
+ if (random().nextInt(20) == 0) {
+ riw.deleteDocuments(new Term("id", Integer.toString(i)));
+ }
+ if (random().nextInt(15) == 0) {
+ riw.updateNumericDocValue(new Term("id"), "dv", Long.valueOf(i));
+ }
}
riw.close();
checkHeaders(dir, new HashMap<String,String>());
@@ -81,25 +77,25 @@ public class TestAllFilesHaveCodecHeader
}
private void checkHeaders(Directory dir, Map<String,String> namesToExtensions) throws IOException {
- SegmentInfos sis = new SegmentInfos();
- sis.read(dir);
- checkHeader(dir, sis.getSegmentsFileName(), namesToExtensions);
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
+ checkHeader(dir, sis.getSegmentsFileName(), namesToExtensions, sis.getId());
for (SegmentCommitInfo si : sis) {
+ assertNotNull(si.info.getId());
for (String file : si.files()) {
- checkHeader(dir, file, namesToExtensions);
+ checkHeader(dir, file, namesToExtensions, si.info.getId());
}
if (si.info.getUseCompoundFile()) {
try (Directory cfsDir = si.info.getCodec().compoundFormat().getCompoundReader(dir, si.info, newIOContext(random()))) {
for (String cfsFile : cfsDir.listAll()) {
- checkHeader(cfsDir, cfsFile, namesToExtensions);
+ checkHeader(cfsDir, cfsFile, namesToExtensions, si.info.getId());
}
}
}
}
}
- private void checkHeader(Directory dir, String file, Map<String,String> namesToExtensions) throws IOException {
+ private void checkHeader(Directory dir, String file, Map<String,String> namesToExtensions, byte[] id) throws IOException {
try (IndexInput in = dir.openInput(file, newIOContext(random()))) {
int val = in.readInt();
assertEquals(file + " has no codec header, instead found: " + val, CodecUtil.CODEC_MAGIC, val);
@@ -114,6 +110,10 @@ public class TestAllFilesHaveCodecHeader
if (previous != null && !previous.equals(extension)) {
fail("extensions " + previous + " and " + extension + " share same codecName " + codecName);
}
+ // read version
+ in.readInt();
+ // read object id
+ CodecUtil.checkIndexHeaderID(in, id);
}
}
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java Fri Oct 24 02:43:41 2014
@@ -199,12 +199,12 @@ public class TestCodecUtil extends Lucen
public void testSegmentHeaderLength() throws Exception {
RAMFile file = new RAMFile();
IndexOutput output = new RAMOutputStream(file, true);
- CodecUtil.writeSegmentHeader(output, "FooBar", 5, StringHelper.randomId(), "xyz");
+ CodecUtil.writeIndexHeader(output, "FooBar", 5, StringHelper.randomId(), "xyz");
output.writeString("this is the data");
output.close();
IndexInput input = new RAMInputStream("file", file);
- input.seek(CodecUtil.segmentHeaderLength("FooBar", "xyz"));
+ input.seek(CodecUtil.indexHeaderLength("FooBar", "xyz"));
assertEquals("this is the data", input.readString());
input.close();
}
@@ -217,7 +217,7 @@ public class TestCodecUtil extends Lucen
RAMFile file = new RAMFile();
IndexOutput output = new RAMOutputStream(file, true);
try {
- CodecUtil.writeSegmentHeader(output, "foobar", 5, StringHelper.randomId(), tooLong.toString());
+ CodecUtil.writeIndexHeader(output, "foobar", 5, StringHelper.randomId(), tooLong.toString());
fail("didn't get expected exception");
} catch (IllegalArgumentException expected) {
// expected
@@ -232,13 +232,13 @@ public class TestCodecUtil extends Lucen
RAMFile file = new RAMFile();
IndexOutput output = new RAMOutputStream(file, true);
byte[] id = StringHelper.randomId();
- CodecUtil.writeSegmentHeader(output, "foobar", 5, id, justLongEnough.toString());
+ CodecUtil.writeIndexHeader(output, "foobar", 5, id, justLongEnough.toString());
output.close();
IndexInput input = new RAMInputStream("file", file);
- CodecUtil.checkSegmentHeader(input, "foobar", 5, 5, id, justLongEnough.toString());
+ CodecUtil.checkIndexHeader(input, "foobar", 5, 5, id, justLongEnough.toString());
assertEquals(input.getFilePointer(), input.length());
- assertEquals(input.getFilePointer(), CodecUtil.segmentHeaderLength("foobar", justLongEnough.toString()));
+ assertEquals(input.getFilePointer(), CodecUtil.indexHeaderLength("foobar", justLongEnough.toString()));
input.close();
}
@@ -246,7 +246,7 @@ public class TestCodecUtil extends Lucen
RAMFile file = new RAMFile();
IndexOutput output = new RAMOutputStream(file, true);
try {
- CodecUtil.writeSegmentHeader(output, "foobar", 5, StringHelper.randomId(), "\u1234");
+ CodecUtil.writeIndexHeader(output, "foobar", 5, StringHelper.randomId(), "\u1234");
fail("didn't get expected exception");
} catch (IllegalArgumentException expected) {
// expected
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java?rev=1633991&r1=1633990&r2=1633991&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java Fri Oct 24 02:43:41 2014
@@ -64,8 +64,7 @@ public class TestConsistentFieldNumbers
writer.close();
- SegmentInfos sis = new SegmentInfos();
- sis.read(dir);
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
assertEquals(2, sis.size());
FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
@@ -82,8 +81,7 @@ public class TestConsistentFieldNumbers
writer.forceMerge(1);
writer.close();
- sis = new SegmentInfos();
- sis.read(dir);
+ sis = SegmentInfos.readLatestCommit(dir);
assertEquals(1, sis.size());
FieldInfos fis3 = IndexWriter.readFieldInfos(sis.info(0));
@@ -130,8 +128,7 @@ public class TestConsistentFieldNumbers
writer.addIndexes(dir2);
writer.close();
- SegmentInfos sis = new SegmentInfos();
- sis.read(dir1);
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir1);
assertEquals(2, sis.size());
FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
@@ -161,8 +158,7 @@ public class TestConsistentFieldNumbers
d.add(new TextField("f2", "d1 second field", Field.Store.YES));
writer.addDocument(d);
writer.close();
- SegmentInfos sis = new SegmentInfos();
- sis.read(dir);
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
assertEquals(1, sis.size());
FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
assertEquals("f1", fis1.fieldInfo(0).name);
@@ -178,8 +174,7 @@ public class TestConsistentFieldNumbers
d.add(new StoredField("f3", new byte[] { 1, 2, 3 }));
writer.addDocument(d);
writer.close();
- SegmentInfos sis = new SegmentInfos();
- sis.read(dir);
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
assertEquals(2, sis.size());
FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
FieldInfos fis2 = IndexWriter.readFieldInfos(sis.info(1));
@@ -199,8 +194,7 @@ public class TestConsistentFieldNumbers
d.add(new StoredField("f3", new byte[] { 1, 2, 3, 4, 5 }));
writer.addDocument(d);
writer.close();
- SegmentInfos sis = new SegmentInfos();
- sis.read(dir);
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
assertEquals(3, sis.size());
FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
FieldInfos fis2 = IndexWriter.readFieldInfos(sis.info(1));
@@ -231,8 +225,7 @@ public class TestConsistentFieldNumbers
writer.forceMerge(1);
writer.close();
- SegmentInfos sis = new SegmentInfos();
- sis.read(dir);
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
assertEquals(1, sis.size());
FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
assertEquals("f1", fis1.fieldInfo(0).name);
@@ -269,8 +262,7 @@ public class TestConsistentFieldNumbers
writer.forceMerge(1);
writer.close();
- SegmentInfos sis = new SegmentInfos();
- sis.read(dir);
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
for (SegmentCommitInfo si : sis) {
FieldInfos fis = IndexWriter.readFieldInfos(si);