You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2013/01/18 19:31:23 UTC
svn commit: r1435287 [7/41] - in /lucene/dev/branches/LUCENE-2878: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/analysis/icu/ dev-tools/maven/
dev-tools/maven/lucene/benchmark/ dev-tools/maven/solr/ dev-tools/m...
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java Fri Jan 18 18:30:54 2013
@@ -71,18 +71,27 @@ public abstract class TermVectorsWriter
* has no vector fields, in this case <code>numVectorFields</code>
* will be zero. */
public abstract void startDocument(int numVectorFields) throws IOException;
-
+
+ /** Called after a doc and all its fields have been added. */
+ public void finishDocument() throws IOException {};
+
/** Called before writing the terms of the field.
* {@link #startTerm(BytesRef, int)} will be called <code>numTerms</code> times. */
public abstract void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads) throws IOException;
-
+
+ /** Called after a field and all its terms have been added. */
+ public void finishField() throws IOException {};
+
/** Adds a term and its term frequency <code>freq</code>.
* If this field has positions and/or offsets enabled, then
* {@link #addPosition(int, int, int, BytesRef)} will be called
* <code>freq</code> times respectively.
*/
public abstract void startTerm(BytesRef term, int freq) throws IOException;
-
+
+ /** Called after a term and all its positions have been added. */
+ public void finishTerm() throws IOException {}
+
/** Adds a term position and offsets */
public abstract void addPosition(int position, int startOffset, int endOffset, BytesRef payload) throws IOException;
@@ -97,7 +106,7 @@ public abstract class TermVectorsWriter
* check that this is the case to detect the JRE bug described
* in LUCENE-1282. */
public abstract void finish(FieldInfos fis, int numDocs) throws IOException;
-
+
/**
* Called by IndexWriter when writing new segments.
* <p>
@@ -197,6 +206,7 @@ public abstract class TermVectorsWriter
protected final void addAllDocVectors(Fields vectors, MergeState mergeState) throws IOException {
if (vectors == null) {
startDocument(0);
+ finishDocument();
return;
}
@@ -275,15 +285,19 @@ public abstract class TermVectorsWriter
addPosition(pos, startOffset, endOffset, payload);
}
}
+ finishTerm();
}
assert termCount == numTerms;
+ finishField();
}
assert fieldCount == numFields;
+ finishDocument();
}
/** Return the BytesRef Comparator used to sort terms
* before feeding to this API. */
public abstract Comparator<BytesRef> getComparator() throws IOException;
+ @Override
public abstract void close() throws IOException;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java Fri Jan 18 18:30:54 2013
@@ -23,6 +23,7 @@ import java.util.Comparator;
import org.apache.lucene.index.FieldInfo; // javadocs
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.MultiDocsEnum;
@@ -103,7 +104,7 @@ public abstract class TermsConsumer {
while((term = termsEnum.next()) != null) {
// We can pass null for liveDocs, because the
// mapping enum will skip the non-live docs:
- docsEnumIn = (MultiDocsEnum) termsEnum.docs(null, docsEnumIn, 0);
+ docsEnumIn = (MultiDocsEnum) termsEnum.docs(null, docsEnumIn, DocsEnum.FLAG_NONE);
if (docsEnumIn != null) {
docsEnum.reset(docsEnumIn);
final PostingsConsumer postingsConsumer = startTerm(term);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java Fri Jan 18 18:30:54 2013
@@ -48,16 +48,30 @@ import org.apache.lucene.store.IOContext
public class CompressingStoredFieldsFormat extends StoredFieldsFormat {
private final String formatName;
+ private final String segmentSuffix;
private final CompressionMode compressionMode;
private final int chunkSize;
/**
+ * Create a new {@link CompressingStoredFieldsFormat} with an empty segment
+ * suffix.
+ *
+ * @see CompressingStoredFieldsFormat#CompressingStoredFieldsFormat(String, String, CompressionMode, int)
+ */
+ public CompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode, int chunkSize) {
+ this(formatName, "", compressionMode, chunkSize);
+ }
+
+ /**
* Create a new {@link CompressingStoredFieldsFormat}.
* <p>
* <code>formatName</code> is the name of the format. This name will be used
* in the file formats to perform
* {@link CodecUtil#checkHeader(org.apache.lucene.store.DataInput, String, int, int) codec header checks}.
* <p>
+ * <code>segmentSuffix</code> is the segment suffix. This suffix is added to
+ * the result file name only if it's not the empty string.
+ * <p>
* The <code>compressionMode</code> parameter allows you to choose between
* compression algorithms that have various compression and decompression
* speeds so that you can pick the one that best fits your indexing and
@@ -81,25 +95,29 @@ public class CompressingStoredFieldsForm
* @param chunkSize the minimum number of bytes of a single chunk of stored documents
* @see CompressionMode
*/
- public CompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode, int chunkSize) {
+ public CompressingStoredFieldsFormat(String formatName, String segmentSuffix,
+ CompressionMode compressionMode, int chunkSize) {
this.formatName = formatName;
+ this.segmentSuffix = segmentSuffix;
this.compressionMode = compressionMode;
if (chunkSize < 1) {
throw new IllegalArgumentException("chunkSize must be >= 1");
}
this.chunkSize = chunkSize;
+
}
@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si,
FieldInfos fn, IOContext context) throws IOException {
- return new CompressingStoredFieldsReader(directory, si, fn, context, formatName, compressionMode);
+ return new CompressingStoredFieldsReader(directory, si, segmentSuffix, fn,
+ context, formatName, compressionMode);
}
@Override
public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si,
IOContext context) throws IOException {
- return new CompressingStoredFieldsWriter(directory, si, context,
+ return new CompressingStoredFieldsWriter(directory, si, segmentSuffix, context,
formatName, compressionMode, chunkSize);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java Fri Jan 18 18:30:54 2013
@@ -54,7 +54,11 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.packed.PackedInts;
-final class CompressingStoredFieldsReader extends StoredFieldsReader {
+/**
+ * {@link StoredFieldsReader} impl for {@link CompressingStoredFieldsFormat}.
+ * @lucene.experimental
+ */
+public final class CompressingStoredFieldsReader extends StoredFieldsReader {
private final FieldInfos fieldInfos;
private final CompressingStoredFieldsIndexReader indexReader;
@@ -79,7 +83,8 @@ final class CompressingStoredFieldsReade
this.closed = false;
}
- public CompressingStoredFieldsReader( Directory d, SegmentInfo si, FieldInfos fn,
+ /** Sole constructor. */
+ public CompressingStoredFieldsReader(Directory d, SegmentInfo si, String segmentSuffix, FieldInfos fn,
IOContext context, String formatName, CompressionMode compressionMode) throws IOException {
this.compressionMode = compressionMode;
final String segment = si.name;
@@ -88,8 +93,8 @@ final class CompressingStoredFieldsReade
numDocs = si.getDocCount();
IndexInput indexStream = null;
try {
- fieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION), context);
- final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION);
+ fieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION), context);
+ final String indexStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION);
indexStream = d.openInput(indexStreamFN, context);
final String codecNameIdx = formatName + CODEC_SFX_IDX;
@@ -123,6 +128,9 @@ final class CompressingStoredFieldsReade
}
}
+ /**
+ * Close the underlying {@link IndexInput}s.
+ */
@Override
public void close() throws IOException {
if (!closed) {
@@ -387,8 +395,10 @@ final class CompressingStoredFieldsReade
* Copy compressed data.
*/
void copyCompressedData(DataOutput out) throws IOException {
- final int chunkSize = chunkSize();
- decompressor.copyCompressedData(fieldsStream, chunkSize, out);
+ final long chunkEnd = docBase + chunkDocs == numDocs
+ ? fieldsStream.length()
+ : indexReader.getStartPointer(docBase + chunkDocs);
+ out.copyBytes(fieldsStream, chunkEnd - fieldsStream.getFilePointer());
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java Fri Jan 18 18:30:54 2013
@@ -47,7 +47,11 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.packed.PackedInts;
-final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
+/**
+ * {@link StoredFieldsWriter} impl for {@link CompressingStoredFieldsFormat}.
+ * @lucene.experimental
+ */
+public final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
static final int STRING = 0x00;
static final int BYTE_ARR = 0x01;
@@ -66,6 +70,7 @@ final class CompressingStoredFieldsWrite
private final Directory directory;
private final String segment;
+ private final String segmentSuffix;
private CompressingStoredFieldsIndexWriter indexWriter;
private IndexOutput fieldsStream;
@@ -79,11 +84,13 @@ final class CompressingStoredFieldsWrite
private int docBase; // doc ID at the beginning of the chunk
private int numBufferedDocs; // docBase + numBufferedDocs == current doc ID
- public CompressingStoredFieldsWriter(Directory directory, SegmentInfo si, IOContext context,
+ /** Sole constructor. */
+ public CompressingStoredFieldsWriter(Directory directory, SegmentInfo si, String segmentSuffix, IOContext context,
String formatName, CompressionMode compressionMode, int chunkSize) throws IOException {
assert directory != null;
this.directory = directory;
this.segment = si.name;
+ this.segmentSuffix = segmentSuffix;
this.compressionMode = compressionMode;
this.compressor = compressionMode.newCompressor();
this.chunkSize = chunkSize;
@@ -94,9 +101,9 @@ final class CompressingStoredFieldsWrite
this.numBufferedDocs = 0;
boolean success = false;
- IndexOutput indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION), context);
+ IndexOutput indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION), context);
try {
- fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION), context);
+ fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION), context);
final String codecNameIdx = formatName + CODEC_SFX_IDX;
final String codecNameDat = formatName + CODEC_SFX_DAT;
@@ -129,19 +136,8 @@ final class CompressingStoredFieldsWrite
}
}
- private void endWithPreviousDocument() throws IOException {
- if (numBufferedDocs > 0) {
- endOffsets[numBufferedDocs - 1] = bufferedDocs.length;
- }
- }
-
@Override
public void startDocument(int numStoredFields) throws IOException {
- endWithPreviousDocument();
- if (triggerFlush()) {
- flush();
- }
-
if (numBufferedDocs == this.numStoredFields.length) {
final int newLength = ArrayUtil.oversize(numBufferedDocs + 1, 4);
this.numStoredFields = Arrays.copyOf(this.numStoredFields, newLength);
@@ -151,6 +147,14 @@ final class CompressingStoredFieldsWrite
++numBufferedDocs;
}
+ @Override
+ public void finishDocument() throws IOException {
+ endOffsets[numBufferedDocs - 1] = bufferedDocs.length;
+ if (triggerFlush()) {
+ flush();
+ }
+ }
+
private static void saveInts(int[] values, int length, DataOutput out) throws IOException {
assert length > 0;
if (length == 1) {
@@ -282,15 +286,16 @@ final class CompressingStoredFieldsWrite
public void abort() {
IOUtils.closeWhileHandlingException(this);
IOUtils.deleteFilesIgnoringExceptions(directory,
- IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION),
- IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION));
+ IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION),
+ IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION));
}
@Override
public void finish(FieldInfos fis, int numDocs) throws IOException {
- endWithPreviousDocument();
if (numBufferedDocs > 0) {
flush();
+ } else {
+ assert bufferedDocs.length == 0;
}
if (docBase != numDocs) {
throw new RuntimeException("Wrote " + docBase + " docs, finish called with numDocs=" + numDocs);
@@ -344,17 +349,13 @@ final class CompressingStoredFieldsWrite
}
if (compressionMode == matchingFieldsReader.getCompressionMode() // same compression mode
- && (numBufferedDocs == 0 || triggerFlush()) // starting a new chunk
+ && numBufferedDocs == 0 // starting a new chunk
&& startOffsets[it.chunkDocs - 1] < chunkSize // chunk is small enough
&& startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] >= chunkSize // chunk is large enough
&& nextDeletedDoc(it.docBase, liveDocs, it.docBase + it.chunkDocs) == it.docBase + it.chunkDocs) { // no deletion in the chunk
assert docID == it.docBase;
// no need to decompress, just copy data
- endWithPreviousDocument();
- if (triggerFlush()) {
- flush();
- }
indexWriter.writeIndex(it.chunkDocs, fieldsStream.getFilePointer());
writeHeader(this.docBase, it.chunkDocs, it.numStoredFields, it.lengths);
it.copyCompressedData(fieldsStream);
@@ -373,6 +374,7 @@ final class CompressingStoredFieldsWrite
final int diff = docID - it.docBase;
startDocument(it.numStoredFields[diff]);
bufferedDocs.writeBytes(it.bytes.bytes, it.bytes.offset + startOffsets[diff], it.lengths[diff]);
+ finishDocument();
++docCount;
mergeState.checkAbort.work(300);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java Fri Jan 18 18:30:54 2013
@@ -141,14 +141,6 @@ public abstract class CompressionMode {
}
@Override
- public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
- final int copied = LZ4.copyCompressedData(in, originalLength, out);
- if (copied != originalLength) {
- throw new CorruptIndexException("Currupted compressed stream: expected " + originalLength + " bytes, but got at least" + copied);
- }
- }
-
- @Override
public Decompressor clone() {
return this;
}
@@ -225,13 +217,6 @@ public abstract class CompressionMode {
}
@Override
- public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
- final int compressedLength = in.readVInt();
- out.writeVInt(compressedLength);
- out.copyBytes(in, compressedLength);
- }
-
- @Override
public Decompressor clone() {
return new DeflateDecompressor();
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java Fri Jan 18 18:30:54 2013
@@ -24,7 +24,10 @@ import org.apache.lucene.store.DataOutpu
/**
* A data compressor.
*/
-abstract class Compressor {
+public abstract class Compressor {
+
+ /** Sole constructor, typically called from sub-classes. */
+ protected Compressor() {}
/**
* Compress bytes into <code>out</code>. It it the responsibility of the
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java Fri Jan 18 18:30:54 2013
@@ -20,13 +20,15 @@ package org.apache.lucene.codecs.compres
import java.io.IOException;
import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.BytesRef;
/**
- * An decompressor.
+ * A decompressor.
*/
-abstract class Decompressor implements Cloneable {
+public abstract class Decompressor implements Cloneable {
+
+ /** Sole constructor, typically called from sub-classes. */
+ protected Decompressor() {}
/**
* Decompress bytes that were stored between offsets <code>offset</code> and
@@ -44,10 +46,6 @@ abstract class Decompressor implements C
*/
public abstract void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException;
- /** Copy a compressed stream whose original length is
- * <code>originalLength</code> from <code>in</code> to <code>out</code>. */
- public abstract void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException;
-
@Override
public abstract Decompressor clone();
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java Fri Jan 18 18:30:54 2013
@@ -121,7 +121,7 @@ class LZ4 {
matchLen += MIN_MATCH;
// copying a multiple of 8 bytes can make decompression from 5% to 10% faster
- final int fastLen = ((matchLen - 1) & 0xFFFFFFF8) + 8;
+ final int fastLen = (matchLen + 7) & 0xFFFFFFF8;
if (matchDec < matchLen || dOff + fastLen > destEnd) {
// overlap -> naive incremental copy
for (int ref = dOff - matchDec, end = dOff + matchLen; dOff < end; ++ref, ++dOff) {
@@ -222,7 +222,7 @@ class LZ4 {
}
// compute match length
- final int matchLen = MIN_MATCH + commonBytes(bytes, ref + 4, off + 4, limit);
+ final int matchLen = MIN_MATCH + commonBytes(bytes, ref + MIN_MATCH, off + MIN_MATCH, limit);
encodeSequence(bytes, anchor, ref, off, matchLen, out);
off += matchLen;
@@ -506,51 +506,4 @@ class LZ4 {
encodeLastLiterals(src, anchor, srcEnd - anchor, out);
}
- /** Copy bytes from <code>in</code> to <code>out</code> where
- * <code>in</code> is a LZ4-encoded stream. This method copies enough bytes
- * so that <code>out</code> can be used later on to restore the first
- * <code>length</code> bytes of the stream. This method always reads at
- * least one byte from <code>in</code> so make sure not to call this method
- * if <code>in</code> reached the end of the stream, even if
- * <code>length=0</code>. */
- public static int copyCompressedData(DataInput in, int length, DataOutput out) throws IOException {
- int n = 0;
- do {
- // literals
- final byte token = in.readByte();
- out.writeByte(token);
- int literalLen = (token & 0xFF) >>> 4;
- if (literalLen == 0x0F) {
- byte len;
- while ((len = in.readByte()) == (byte) 0xFF) {
- literalLen += 0xFF;
- out.writeByte(len);
- }
- literalLen += len & 0xFF;
- out.writeByte(len);
- }
- out.copyBytes(in, literalLen);
- n += literalLen;
- if (n >= length) {
- break;
- }
-
- // matchs
- out.copyBytes(in, 2); // match dec
- int matchLen = token & 0x0F;
- if (matchLen == 0x0F) {
- byte len;
- while ((len = in.readByte()) == (byte) 0xFF) {
- matchLen += 0xFF;
- out.writeByte(len);
- }
- matchLen += len & 0xFF;
- out.writeByte(len);
- }
- matchLen += MIN_MATCH;
- n += matchLen;
- } while (n < length);
- return n;
- }
-
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java Fri Jan 18 18:30:54 2013
@@ -26,6 +26,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.MutableBits;
/** Optimized implementation of a vector of bits. This is more-or-less like
@@ -109,6 +110,7 @@ final class BitVector implements Cloneab
}
/** Sets the value of <code>bit</code> to zero. */
+ @Override
public final void clear(int bit) {
if (bit >= size) {
throw new ArrayIndexOutOfBoundsException(bit);
@@ -138,6 +140,7 @@ final class BitVector implements Cloneab
/** Returns <code>true</code> if <code>bit</code> is one and
<code>false</code> if it is zero. */
+ @Override
public final boolean get(int bit) {
assert bit >= 0 && bit < size: "bit " + bit + " is out of bounds 0.." + (size-1);
return (bits[bit >> 3] & (1 << (bit & 7))) != 0;
@@ -236,7 +239,7 @@ final class BitVector implements Cloneab
}
assert verifyCount();
} finally {
- output.close();
+ IOUtils.close(output);
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java Fri Jan 18 18:30:54 2013
@@ -124,6 +124,7 @@ public final class Lucene40StoredFieldsR
*
* @throws IOException If an I/O error occurs
*/
+ @Override
public final void close() throws IOException {
if (!closed) {
IOUtils.close(fieldsStream, indexStream);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java Fri Jan 18 18:30:54 2013
@@ -113,11 +113,13 @@ public final class Lucene40StoredFieldsW
// and adds a new entry for this document into the index
// stream. This assumes the buffer was already written
// in the correct fields format.
+ @Override
public void startDocument(int numStoredFields) throws IOException {
indexStream.writeLong(fieldsStream.getFilePointer());
fieldsStream.writeVInt(numStoredFields);
}
+ @Override
public void close() throws IOException {
try {
IOUtils.close(fieldsStream, indexStream);
@@ -126,6 +128,7 @@ public final class Lucene40StoredFieldsW
}
}
+ @Override
public void abort() {
try {
close();
@@ -135,6 +138,7 @@ public final class Lucene40StoredFieldsW
IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION));
}
+ @Override
public void writeField(FieldInfo info, StorableField field) throws IOException {
fieldsStream.writeVInt(info.number);
int bits = 0;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java Fri Jan 18 18:30:54 2013
@@ -124,17 +124,16 @@ public final class Lucene40TermVectorsWr
if (payloads)
bits |= Lucene40TermVectorsReader.STORE_PAYLOAD_WITH_TERMVECTOR;
tvf.writeByte(bits);
-
- assert fieldCount <= numVectorFields;
- if (fieldCount == numVectorFields) {
- // last field of the document
- // this is crazy because the file format is crazy!
- for (int i = 1; i < fieldCount; i++) {
- tvd.writeVLong(fps[i] - fps[i-1]);
- }
- }
}
+ @Override
+ public void finishDocument() throws IOException {
+ assert fieldCount == numVectorFields;
+ for (int i = 1; i < fieldCount; i++) {
+ tvd.writeVLong(fps[i] - fps[i-1]);
+ }
+ }
+
private final BytesRef lastTerm = new BytesRef(10);
// NOTE: we override addProx, so we don't need to buffer when indexing.
@@ -222,20 +221,6 @@ public final class Lucene40TermVectorsWr
}
bufferedIndex++;
-
- // dump buffer if we are done
- if (bufferedIndex == bufferedFreq) {
- if (payloads) {
- tvf.writeBytes(payloadData.bytes, payloadData.offset, payloadData.length);
- }
- for (int i = 0; i < bufferedIndex; i++) {
- if (offsets) {
- tvf.writeVInt(offsetStartBuffer[i] - lastOffset);
- tvf.writeVInt(offsetEndBuffer[i] - offsetStartBuffer[i]);
- lastOffset = offsetEndBuffer[i];
- }
- }
- }
} else if (positions) {
// write position delta
writePosition(position - lastPosition, payload);
@@ -248,6 +233,25 @@ public final class Lucene40TermVectorsWr
}
}
+ @Override
+ public void finishTerm() throws IOException {
+ if (bufferedIndex > 0) {
+ // dump buffer
+ assert positions && (offsets || payloads);
+ assert bufferedIndex == bufferedFreq;
+ if (payloads) {
+ tvf.writeBytes(payloadData.bytes, payloadData.offset, payloadData.length);
+ }
+ for (int i = 0; i < bufferedIndex; i++) {
+ if (offsets) {
+ tvf.writeVInt(offsetStartBuffer[i] - lastOffset);
+ tvf.writeVInt(offsetEndBuffer[i] - offsetStartBuffer[i]);
+ lastOffset = offsetEndBuffer[i];
+ }
+ }
+ }
+ }
+
private void writePosition(int delta, BytesRef payload) throws IOException {
if (payloads) {
int payloadLength = payload == null ? 0 : payload.length;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java Fri Jan 18 18:30:54 2013
@@ -469,6 +469,7 @@ public final class Bytes {
}
}
+ @Override
public int getValueSize() {
return size;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DirectSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DirectSource.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DirectSource.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DirectSource.java Fri Jan 18 18:30:54 2013
@@ -130,6 +130,7 @@ abstract class DirectSource extends Sour
throw new UnsupportedOperationException("ints are not supported");
}
+ @Override
double toDouble(IndexInput input) throws IOException {
return Float.intBitsToFloat(input.readInt());
}
@@ -141,6 +142,7 @@ abstract class DirectSource extends Sour
throw new UnsupportedOperationException("ints are not supported");
}
+ @Override
double toDouble(IndexInput input) throws IOException {
return Double.longBitsToDouble(input.readLong());
}
@@ -153,6 +155,7 @@ abstract class DirectSource extends Sour
return input.readLong();
}
+ @Override
double toDouble(IndexInput input) {
throw new UnsupportedOperationException("doubles are not supported");
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java Fri Jan 18 18:30:54 2013
@@ -122,7 +122,7 @@ public final class Lucene41PostingsReade
/**
* Read values that have been written using variable-length encoding instead of bit-packing.
*/
- private static void readVIntBlock(IndexInput docIn, int[] docBuffer,
+ static void readVIntBlock(IndexInput docIn, int[] docBuffer,
int[] freqBuffer, int num, boolean indexHasFreq) throws IOException {
if (indexHasFreq) {
for(int i=0;i<num;i++) {
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/package.html?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/package.html (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/package.html Fri Jan 18 18:30:54 2013
@@ -147,7 +147,7 @@ its numbering.</p>
This contains the set of field names used in the index.
</li>
<li>
-{@link org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat Stored Field values}.
+{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Stored Field values}.
This contains, for each document, a list of attribute-value pairs, where the attributes
are field names. These are used to store auxiliary information about the document, such as
its title, url, or an identifier to access a database. The set of stored fields are what is
@@ -254,12 +254,12 @@ systems that frequently run out of file
<td>Stores information about the fields</td>
</tr>
<tr>
-<td>{@link org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat Field Index}</td>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Field Index}</td>
<td>.fdx</td>
<td>Contains pointers to field data</td>
</tr>
<tr>
-<td>{@link org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat Field Data}</td>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Field Data}</td>
<td>.fdt</td>
<td>The stored fields for documents</td>
</tr>
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java Fri Jan 18 18:30:54 2013
@@ -39,12 +39,15 @@ public class DocumentStoredFieldVisitor
private final StoredDocument doc = new StoredDocument();
private final Set<String> fieldsToAdd;
- /** Load only fields named in the provided <code>Set<String></code>. */
+ /**
+ * Load only fields named in the provided <code>Set<String></code>.
+ * @param fieldsToAdd Set of fields to load, or <code>null</code> (all fields).
+ */
public DocumentStoredFieldVisitor(Set<String> fieldsToAdd) {
this.fieldsToAdd = fieldsToAdd;
}
- /** Load only fields named in the provided <code>Set<String></code>. */
+ /** Load only fields named in the provided fields. */
public DocumentStoredFieldVisitor(String... fields) {
fieldsToAdd = new HashSet<String>(fields.length);
for(String field : fields) {
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/Field.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/Field.java Fri Jan 18 18:30:54 2013
@@ -266,6 +266,7 @@ public class Field implements IndexableF
* binary value is used. Exactly one of stringValue(), readerValue(), and
* getBinaryValue() must be set.
*/
+ @Override
public String stringValue() {
if (fieldsData instanceof String || fieldsData instanceof Number) {
return fieldsData.toString();
@@ -279,6 +280,7 @@ public class Field implements IndexableF
* binary value is used. Exactly one of stringValue(), readerValue(), and
* getBinaryValue() must be set.
*/
+ @Override
public Reader readerValue() {
return fieldsData instanceof Reader ? (Reader) fieldsData : null;
}
@@ -345,7 +347,7 @@ public class Field implements IndexableF
throw new IllegalArgumentException("cannot change value type from " + fieldsData.getClass().getSimpleName() + " to BytesRef");
}
if (type.indexed()) {
- throw new IllegalArgumentException("cannot set a Reader value on an indexed field");
+ throw new IllegalArgumentException("cannot set a BytesRef value on an indexed field");
}
fieldsData = value;
}
@@ -431,6 +433,7 @@ public class Field implements IndexableF
this.tokenStream = tokenStream;
}
+ @Override
public String name() {
return name;
}
@@ -441,6 +444,7 @@ public class Field implements IndexableF
* The default value is <code>1.0f</code> (no boost).
* @see #setBoost(float)
*/
+ @Override
public float boost() {
return boost;
}
@@ -460,6 +464,7 @@ public class Field implements IndexableF
this.boost = boost;
}
+ @Override
public Number numericValue() {
if (fieldsData instanceof Number) {
return (Number) fieldsData;
@@ -468,6 +473,7 @@ public class Field implements IndexableF
}
}
+ @Override
public BytesRef binaryValue() {
if (fieldsData instanceof BytesRef) {
return (BytesRef) fieldsData;
@@ -494,10 +500,12 @@ public class Field implements IndexableF
}
/** Returns the {@link FieldType} for this field. */
+ @Override
public FieldType fieldType() {
return type;
}
+ @Override
public TokenStream tokenStream(Analyzer analyzer) throws IOException {
if (!fieldType().indexed()) {
return null;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/FieldType.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/FieldType.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/FieldType.java Fri Jan 18 18:30:54 2013
@@ -102,6 +102,7 @@ public class FieldType implements Indexa
* The default is <code>false</code>.
* @see #setIndexed(boolean)
*/
+ @Override
public boolean indexed() {
return this.indexed;
}
@@ -124,6 +125,7 @@ public class FieldType implements Indexa
* The default is <code>false</code>.
* @see #setStored(boolean)
*/
+ @Override
public boolean stored() {
return this.stored;
}
@@ -146,6 +148,7 @@ public class FieldType implements Indexa
* The default is <code>true</code>.
* @see #setTokenized(boolean)
*/
+ @Override
public boolean tokenized() {
return this.tokenized;
}
@@ -169,6 +172,7 @@ public class FieldType implements Indexa
* The default is <code>false</code>.
* @see #setStoreTermVectors(boolean)
*/
+ @Override
public boolean storeTermVectors() {
return this.storeTermVectors;
}
@@ -192,6 +196,7 @@ public class FieldType implements Indexa
* The default is <code>false</code>.
* @see #setStoreTermVectorOffsets(boolean)
*/
+ @Override
public boolean storeTermVectorOffsets() {
return this.storeTermVectorOffsets;
}
@@ -215,6 +220,7 @@ public class FieldType implements Indexa
* The default is <code>false</code>.
* @see #setStoreTermVectorPositions(boolean)
*/
+ @Override
public boolean storeTermVectorPositions() {
return this.storeTermVectorPositions;
}
@@ -238,6 +244,7 @@ public class FieldType implements Indexa
* The default is <code>false</code>.
* @see #setStoreTermVectorPayloads(boolean)
*/
+ @Override
public boolean storeTermVectorPayloads() {
return this.storeTermVectorPayloads;
}
@@ -261,6 +268,7 @@ public class FieldType implements Indexa
* The default is <code>false</code>.
* @see #setOmitNorms(boolean)
*/
+ @Override
public boolean omitNorms() {
return this.omitNorms;
}
@@ -283,6 +291,7 @@ public class FieldType implements Indexa
* The default is {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS}.
* @see #setIndexOptions(org.apache.lucene.index.FieldInfo.IndexOptions)
*/
+ @Override
public IndexOptions indexOptions() {
return this.indexOptions;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java Fri Jan 18 18:30:54 2013
@@ -90,6 +90,7 @@ public abstract class AtomicReader exten
* field does not exists. This method does not take into
* account deleted documents that have not yet been merged
* away. */
+ @Override
public final long totalTermFreq(Term term) throws IOException {
final Fields fields = fields();
if (fields == null) {
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BitsSlice.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BitsSlice.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BitsSlice.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BitsSlice.java Fri Jan 18 18:30:54 2013
@@ -37,6 +37,7 @@ final class BitsSlice implements Bits {
assert length >= 0: "length=" + length;
}
+ @Override
public boolean get(int doc) {
if (doc >= length) {
throw new RuntimeException("doc " + doc + " is out of bounds 0 .. " + (length-1));
@@ -45,6 +46,7 @@ final class BitsSlice implements Bits {
return parent.get(doc+start);
}
+ @Override
public int length() {
return length;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java Fri Jan 18 18:30:54 2013
@@ -168,6 +168,8 @@ class BufferedDeletesStream {
infoStream.message("BD", "applyDeletes: infos=" + infos + " packetCount=" + deletes.size());
}
+ final long gen = nextGen++;
+
List<SegmentInfoPerCommit> infos2 = new ArrayList<SegmentInfoPerCommit>();
infos2.addAll(infos);
Collections.sort(infos2, sortSegInfoByDelGen);
@@ -255,7 +257,7 @@ class BufferedDeletesStream {
*/
delIDX--;
infosIDX--;
- info.setBufferedDeletesGen(nextGen);
+ info.setBufferedDeletesGen(gen);
} else {
//System.out.println(" gt");
@@ -290,7 +292,7 @@ class BufferedDeletesStream {
infoStream.message("BD", "seg=" + info + " segGen=" + segGen + " coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "] newDelCount=" + delCount + (segAllDeletes ? " 100% deleted" : ""));
}
}
- info.setBufferedDeletesGen(nextGen);
+ info.setBufferedDeletesGen(gen);
infosIDX--;
}
@@ -302,7 +304,7 @@ class BufferedDeletesStream {
}
// assert infos != segmentInfos || !any() : "infos=" + infos + " segmentInfos=" + segmentInfos + " any=" + any;
- return new ApplyDeletesResult(anyNewDeletes, nextGen++, allDeleted);
+ return new ApplyDeletesResult(anyNewDeletes, gen, allDeleted);
}
synchronized long getNextGen() {
@@ -397,7 +399,7 @@ class BufferedDeletesStream {
if (termsEnum.seekExact(term.bytes(), false)) {
// we don't need term frequencies for this
- DocsEnum docsEnum = termsEnum.docs(rld.getLiveDocs(), docs, 0);
+ DocsEnum docsEnum = termsEnum.docs(rld.getLiveDocs(), docs, DocsEnum.FLAG_NONE);
//System.out.println("BDS: got docsEnum=" + docsEnum);
if (docsEnum != null) {
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Fri Jan 18 18:30:54 2013
@@ -905,7 +905,7 @@ public class CheckIndex {
totalTermFreq += docsNoDel.freq();
}
} else {
- final DocsEnum docsNoDel = termsEnum.docs(null, docs, 0);
+ final DocsEnum docsNoDel = termsEnum.docs(null, docs, DocsEnum.FLAG_NONE);
docCount = 0;
totalTermFreq = -1;
while(docsNoDel.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
@@ -991,7 +991,7 @@ public class CheckIndex {
} else {
for(int idx=0;idx<7;idx++) {
final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
- docs = termsEnum.docs(liveDocs, docs, 0);
+ docs = termsEnum.docs(liveDocs, docs, DocsEnum.FLAG_NONE);
final int docID = docs.advance(skipDocID);
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
break;
@@ -1057,7 +1057,7 @@ public class CheckIndex {
}
int expectedDocFreq = termsEnum.docFreq();
- DocsEnum d = termsEnum.docs(null, null, 0);
+ DocsEnum d = termsEnum.docs(null, null, DocsEnum.FLAG_NONE);
int docFreq = 0;
while (d.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
docFreq++;
@@ -1098,7 +1098,7 @@ public class CheckIndex {
throw new RuntimeException("seek to existing term " + seekTerms[i] + " failed");
}
- docs = termsEnum.docs(liveDocs, docs, 0);
+ docs = termsEnum.docs(liveDocs, docs, DocsEnum.FLAG_NONE);
if (docs == null) {
throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]);
}
@@ -1116,7 +1116,7 @@ public class CheckIndex {
}
totDocFreq += termsEnum.docFreq();
- docs = termsEnum.docs(null, docs, 0);
+ docs = termsEnum.docs(null, docs, DocsEnum.FLAG_NONE);
if (docs == null) {
throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Fri Jan 18 18:30:54 2013
@@ -55,14 +55,15 @@ public class ConcurrentMergeScheduler ex
// forcefully pause the larger ones, letting the smaller
// ones run, up until maxMergeCount merges at which point
// we forcefully pause incoming threads (that presumably
- // are the ones causing so much merging). We dynamically
- // default this from 1 to 3, depending on how many cores
- // you have:
- private int maxThreadCount = Math.max(1, Math.min(3, Runtime.getRuntime().availableProcessors()/2));
+ // are the ones causing so much merging). We default to 1
+ // here: tests on spinning-magnet drives showed slower
+ // indexing perf if more than one merge thread runs at
+ // once (though on an SSD it was faster):
+ private int maxThreadCount = 1;
// Max number of merges we accept before forcefully
// throttling the incoming threads
- private int maxMergeCount = maxThreadCount+2;
+ private int maxMergeCount = 2;
/** {@link Directory} that holds the index. */
protected Directory dir;
@@ -144,6 +145,7 @@ public class ConcurrentMergeScheduler ex
/** Sorts {@link MergeThread}s; larger merges come first. */
protected static final Comparator<MergeThread> compareByMergeDocCount = new Comparator<MergeThread>() {
+ @Override
public int compare(MergeThread t1, MergeThread t2) {
final MergePolicy.OneMerge m1 = t1.getCurrentMerge();
final MergePolicy.OneMerge m2 = t2.getCurrentMerge();
@@ -478,6 +480,14 @@ public class ConcurrentMergeScheduler ex
// Subsequent times through the loop we do any new
// merge that writer says is necessary:
merge = tWriter.getNextMerge();
+
+ // Notify here in case any threads were stalled;
+ // they will notice that the pending merge has
+ // been pulled and possibly resume:
+ synchronized(ConcurrentMergeScheduler.this) {
+ ConcurrentMergeScheduler.this.notifyAll();
+ }
+
if (merge != null) {
updateMergeThreads();
if (verbose()) {
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java Fri Jan 18 18:30:54 2013
@@ -327,6 +327,7 @@ final class DocFieldProcessor extends Do
}
private static final Comparator<DocFieldProcessorPerField> fieldsComp = new Comparator<DocFieldProcessorPerField>() {
+ @Override
public int compare(DocFieldProcessorPerField o1, DocFieldProcessorPerField o2) {
return o1.fieldInfo.name.compareTo(o2.fieldInfo.name);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java Fri Jan 18 18:30:54 2013
@@ -106,75 +106,72 @@ final class DocInverterPerField extends
OffsetAttribute offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class);
PositionIncrementAttribute posIncrAttribute = fieldState.attributeSource.addAttribute(PositionIncrementAttribute.class);
- consumer.start(field);
+ if (hasMoreTokens) {
+ consumer.start(field);
- for (;;) {
-
- // If we hit an exception in stream.next below
- // (which is fairly common, eg if analyzer
- // chokes on a given document), then it's
- // non-aborting and (above) this one document
- // will be marked as deleted, but still
- // consume a docID
-
- if (!hasMoreTokens) break;
-
- final int posIncr = posIncrAttribute.getPositionIncrement();
- if (posIncr < 0) {
- throw new IllegalArgumentException("position increment must be >=0 (got " + posIncr + ")");
- }
- if (fieldState.position == 0 && posIncr == 0) {
- throw new IllegalArgumentException("first position increment must be > 0 (got 0)");
- }
- int position = fieldState.position + posIncr;
- if (position > 0) {
- // NOTE: confusing: this "mirrors" the
- // position++ we do below
- position--;
- } else if (position < 0) {
- throw new IllegalArgumentException("position overflow for field '" + field.name() + "'");
- }
-
- // position is legal, we can safely place it in fieldState now.
- // not sure if anything will use fieldState after non-aborting exc...
- fieldState.position = position;
-
- if (posIncr == 0)
- fieldState.numOverlap++;
-
- if (checkOffsets) {
- int startOffset = fieldState.offset + offsetAttribute.startOffset();
- int endOffset = fieldState.offset + offsetAttribute.endOffset();
- if (startOffset < 0 || endOffset < startOffset) {
- throw new IllegalArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset, "
- + "startOffset=" + startOffset + ",endOffset=" + endOffset);
+ do {
+ // If we hit an exception in stream.next below
+ // (which is fairly common, eg if analyzer
+ // chokes on a given document), then it's
+ // non-aborting and (above) this one document
+ // will be marked as deleted, but still
+ // consume a docID
+
+ final int posIncr = posIncrAttribute.getPositionIncrement();
+ if (posIncr < 0) {
+ throw new IllegalArgumentException("position increment must be >=0 (got " + posIncr + ")");
}
- if (startOffset < lastStartOffset) {
- throw new IllegalArgumentException("offsets must not go backwards startOffset="
- + startOffset + " is < lastStartOffset=" + lastStartOffset);
+ if (fieldState.position == 0 && posIncr == 0) {
+ throw new IllegalArgumentException("first position increment must be > 0 (got 0)");
}
- lastStartOffset = startOffset;
- }
-
- boolean success = false;
- try {
- // If we hit an exception in here, we abort
- // all buffered documents since the last
- // flush, on the likelihood that the
- // internal state of the consumer is now
- // corrupt and should not be flushed to a
- // new segment:
- consumer.add();
- success = true;
- } finally {
- if (!success) {
- docState.docWriter.setAborting();
+ int position = fieldState.position + posIncr;
+ if (position > 0) {
+ // NOTE: confusing: this "mirrors" the
+ // position++ we do below
+ position--;
+ } else if (position < 0) {
+ throw new IllegalArgumentException("position overflow for field '" + field.name() + "'");
+ }
+
+ // position is legal, we can safely place it in fieldState now.
+ // not sure if anything will use fieldState after non-aborting exc...
+ fieldState.position = position;
+
+ if (posIncr == 0)
+ fieldState.numOverlap++;
+
+ if (checkOffsets) {
+ int startOffset = fieldState.offset + offsetAttribute.startOffset();
+ int endOffset = fieldState.offset + offsetAttribute.endOffset();
+ if (startOffset < 0 || endOffset < startOffset) {
+ throw new IllegalArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset, "
+ + "startOffset=" + startOffset + ",endOffset=" + endOffset);
+ }
+ if (startOffset < lastStartOffset) {
+ throw new IllegalArgumentException("offsets must not go backwards startOffset="
+ + startOffset + " is < lastStartOffset=" + lastStartOffset);
+ }
+ lastStartOffset = startOffset;
}
- }
- fieldState.length++;
- fieldState.position++;
- hasMoreTokens = stream.incrementToken();
+ boolean success = false;
+ try {
+ // If we hit an exception in here, we abort
+ // all buffered documents since the last
+ // flush, on the likelihood that the
+ // internal state of the consumer is now
+ // corrupt and should not be flushed to a
+ // new segment:
+ consumer.add();
+ success = true;
+ } finally {
+ if (!success) {
+ docState.docWriter.setAborting();
+ }
+ }
+ fieldState.length++;
+ fieldState.position++;
+ } while (stream.incrementToken());
}
// trigger streams to perform end-of-stream operations
stream.end();
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java Fri Jan 18 18:30:54 2013
@@ -363,7 +363,7 @@ public class DocTermOrds {
final int df = te.docFreq();
if (df <= maxTermDocFreq) {
- docsEnum = te.docs(liveDocs, docsEnum, 0);
+ docsEnum = te.docs(liveDocs, docsEnum, DocsEnum.FLAG_NONE);
// dF, but takes deletions into account
int actualDF = 0;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocValues.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocValues.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocValues.java Fri Jan 18 18:30:54 2013
@@ -163,6 +163,7 @@ public abstract class DocValues implemen
* by the creator of this {@link DocValues} instance. API users should not
* close {@link DocValues} instances.
*/
+ @Override
public void close() throws IOException {
cache.close(this);
}
@@ -784,6 +785,7 @@ public abstract class DocValues implemen
public DirectSourceCache() {
}
+ @Override
public synchronized Source load(DocValues values) throws IOException {
if (ref == null) {
ref = values.loadSource();
@@ -791,6 +793,7 @@ public abstract class DocValues implemen
return ref;
}
+ @Override
public synchronized void invalidate(DocValues values) {
ref = null;
directSourceCache.close();
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java Fri Jan 18 18:30:54 2013
@@ -24,6 +24,7 @@ import org.apache.lucene.util.BytesRef;
/** Also iterates through positions. */
public abstract class DocsAndPositionsEnum extends DocsEnum {
+
/** Flag to pass to {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)}
* if you require offsets in the returned enum. */
public static final int FLAG_OFFSETS = 0x1;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java Fri Jan 18 18:30:54 2013
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Bits; // javadocs
@@ -27,6 +28,14 @@ import org.apache.lucene.util.Bits; // j
* NOTE: you must first call {@link #nextDoc} before using
* any of the per-doc methods. */
public abstract class DocsEnum extends DocIdSetIterator {
+
+ /**
+ * Flag to pass to {@link TermsEnum#docs(Bits,DocsEnum,int)} if you don't
+ * require term frequencies in the returned enum. When passed to
+ * {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)} means
+ * that no offsets and payloads will be returned.
+ */
+ public static final int FLAG_NONE = 0x0;
/** Flag to pass to {@link TermsEnum#docs(Bits,DocsEnum,int)}
* if you require term frequencies in the returned enum. */
@@ -39,10 +48,16 @@ public abstract class DocsEnum extends D
protected DocsEnum() {
}
- /** Returns term frequency in the current document. Do
- * not call this before {@link #nextDoc} is first called,
- * nor after {@link #nextDoc} returns NO_MORE_DOCS.
- **/
+ /**
+ * Returns term frequency in the current document, or 1 if the field was
+ * indexed with {@link IndexOptions#DOCS_ONLY}. Do not call this before
+ * {@link #nextDoc} is first called, nor after {@link #nextDoc} returns
+ * {@link DocIdSetIterator#NO_MORE_DOCS}.
+ *
+ * <p>
+ * <b>NOTE:</b> if the {@link DocsEnum} was obtain with {@link #FLAG_NONE},
+ * the result of this method is undefined.
+ */
public abstract int freq() throws IOException;
/** Returns the related attributes. */
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java Fri Jan 18 18:30:54 2013
@@ -384,14 +384,17 @@ final class DocumentsWriterFlushControl
return new Iterator<ThreadState>() {
int i = 0;
+ @Override
public boolean hasNext() {
return i < upto;
}
+ @Override
public ThreadState next() {
return perThreadPool.getThreadState(i++);
}
+ @Override
public void remove() {
throw new UnsupportedOperationException("remove() not supported.");
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java Fri Jan 18 18:30:54 2013
@@ -178,6 +178,7 @@ class DocumentsWriterFlushQueue {
protected GlobalDeletesTicket(FrozenBufferedDeletes frozenDeletes) {
super(frozenDeletes);
}
+ @Override
protected void publish(DocumentsWriter writer) throws IOException {
assert !published : "ticket was already publised - can not publish twice";
published = true;
@@ -185,6 +186,7 @@ class DocumentsWriterFlushQueue {
writer.finishFlush(null, frozenDeletes);
}
+ @Override
protected boolean canPublish() {
return true;
}
@@ -198,6 +200,7 @@ class DocumentsWriterFlushQueue {
super(frozenDeletes);
}
+ @Override
protected void publish(DocumentsWriter writer) throws IOException {
assert !published : "ticket was already publised - can not publish twice";
published = true;
@@ -214,6 +217,7 @@ class DocumentsWriterFlushQueue {
failed = true;
}
+ @Override
protected boolean canPublish() {
return segment != null || failed;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Fri Jan 18 18:30:54 2013
@@ -296,7 +296,9 @@ class DocumentsWriterPerThread {
infoStream.message("DWPT", Thread.currentThread().getName() + " update delTerm=" + delTerm + " docID=" + docState.docID + " seg=" + segmentInfo.name);
}
int docCount = 0;
+ boolean allDocsIndexed = false;
try {
+
for(IndexDocument doc : docs) {
docState.doc = doc;
docState.docID = numDocsInRAM;
@@ -309,20 +311,7 @@ class DocumentsWriterPerThread {
} finally {
if (!success) {
// An exc is being thrown...
-
if (!aborting) {
- // One of the documents hit a non-aborting
- // exception (eg something happened during
- // analysis). We now go and mark any docs
- // from this batch that we had already indexed
- // as deleted:
- int docID = docState.docID;
- final int endDocID = docID - docCount;
- while (docID > endDocID) {
- deleteDocID(docID);
- docID--;
- }
-
// Incr here because finishDocument will not
// be called (because an exc is being thrown):
numDocsInRAM++;
@@ -343,6 +332,7 @@ class DocumentsWriterPerThread {
finishDocument(null);
}
+ allDocsIndexed = true;
// Apply delTerm only after all indexing has
// succeeded, but apply it only to docs prior to when
@@ -354,6 +344,16 @@ class DocumentsWriterPerThread {
}
} finally {
+ if (!allDocsIndexed && !aborting) {
+ // the iterator threw an exception that is not aborting
+ // go and mark all docs from this block as deleted
+ int docID = numDocsInRAM-1;
+ final int endDocID = docID - docCount;
+ while (docID > endDocID) {
+ deleteDocID(docID);
+ docID--;
+ }
+ }
docState.clear();
}
@@ -637,6 +637,7 @@ class DocumentsWriterPerThread {
}
/* Allocate another int[] from the shared pool */
+ @Override
public int[] getIntBlock() {
int[] b = new int[IntBlockPool.INT_BLOCK_SIZE];
bytesUsed.addAndGet(IntBlockPool.INT_BLOCK_SIZE
@@ -644,6 +645,7 @@ class DocumentsWriterPerThread {
return b;
}
+ @Override
public void recycleIntBlocks(int[][] blocks, int offset, int length) {
bytesUsed.addAndGet(-(length * (IntBlockPool.INT_BLOCK_SIZE * RamUsageEstimator.NUM_BYTES_INT)));
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Fri Jan 18 18:30:54 2013
@@ -131,6 +131,7 @@ public class FieldInfos implements Itera
* ordered by ascending field number
*/
// TODO: what happens if in fact a different order is used?
+ @Override
public Iterator<FieldInfo> iterator() {
return values.iterator();
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/Fields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/Fields.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/Fields.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/Fields.java Fri Jan 18 18:30:54 2013
@@ -32,6 +32,7 @@ public abstract class Fields implements
/** Returns an iterator that will step through all fields
* names. This will not return null. */
+ @Override
public abstract Iterator<String> iterator();
/** Get the {@link Terms} for this field. This will return
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java Fri Jan 18 18:30:54 2013
@@ -87,7 +87,7 @@ public class FilterAtomicReader extends
}
@Override
- public Comparator<BytesRef> getComparator() throws IOException {
+ public Comparator<BytesRef> getComparator() {
return in.getComparator();
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Fri Jan 18 18:30:54 2013
@@ -78,6 +78,7 @@ final class FreqProxTermsWriterPerField
@Override
void skippingLongTerm() {}
+ @Override
public int compareTo(FreqProxTermsWriterPerField other) {
return fieldInfo.name.compareTo(other.fieldInfo.name);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java Fri Jan 18 18:30:54 2013
@@ -107,10 +107,11 @@ public abstract class IndexCommit implem
public abstract long getGeneration();
/** Returns userData, previously passed to {@link
- * IndexWriter#commit(Map)} for this commit. Map is
+ * IndexWriter#setCommitData(Map)} for this commit. Map is
* String -> String. */
public abstract Map<String,String> getUserData() throws IOException;
+ @Override
public int compareTo(IndexCommit commit) {
if (getDirectory() != commit.getDirectory()) {
throw new UnsupportedOperationException("cannot compare IndexCommits from different Directory instances");
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java Fri Jan 18 18:30:54 2013
@@ -17,6 +17,7 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
@@ -69,7 +70,7 @@ import org.apache.lucene.util.InfoStream
* directly with no retry logic.
*/
-final class IndexFileDeleter {
+final class IndexFileDeleter implements Closeable {
/* Files that we tried to delete but failed (likely
* because they are open and we are running on Windows),
@@ -435,8 +436,9 @@ final class IndexFileDeleter {
assert locked();
assert Thread.holdsLock(writer);
-
+ long t0 = 0;
if (infoStream.isEnabled("IFD")) {
+ t0 = System.nanoTime();
infoStream.message("IFD", "now checkpoint \"" + writer.segString(writer.toLiveInfos(segmentInfos)) + "\" [" + segmentInfos.size() + " segments " + "; isCommit = " + isCommit + "]");
}
@@ -466,6 +468,10 @@ final class IndexFileDeleter {
// Save files so we can decr on next checkpoint/commit:
lastFiles.add(segmentInfos.files(directory, false));
}
+ if (infoStream.isEnabled("IFD")) {
+ long t1 = System.nanoTime();
+ infoStream.message("IFD", ((t1-t0)/1000000) + " msec to checkpoint");
+ }
}
void incRef(SegmentInfos segmentInfos, boolean isCommit) throws IOException {
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexReader.java Fri Jan 18 18:30:54 2013
@@ -370,6 +370,7 @@ public abstract class IndexReader implem
* No other methods should be called after this has been called.
* @throws IOException if there is a low-level IO error
*/
+ @Override
public final synchronized void close() throws IOException {
if (!closed) {
decRef();