You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2012/10/25 15:10:51 UTC
svn commit: r1402140 [4/17] - in /lucene/dev/branches/LUCENE-2878: ./
dev-tools/ dev-tools/eclipse/ dev-tools/eclipse/dot.settings/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/classification/
dev-tools/maven/ dev-tools/maven/lucene/classifica...
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java Thu Oct 25 13:10:25 2012
@@ -111,7 +111,6 @@ public class FuzzySet {
/**
* Use this method to choose a set size where accuracy (low content saturation) is more important
* than deciding how much memory to throw at the problem.
- * @param maxNumberOfValuesExpected
* @param desiredSaturation A number between 0 and 1 expressing the % of bits set once all values have been recorded
* @return The size of the set nearest to the required size
*/
@@ -155,7 +154,6 @@ public class FuzzySet {
/**
* The main method required for a Bloom filter which, given a value determines set membership.
* Unlike a conventional set, the fuzzy set returns NO or MAYBE rather than true or false.
- * @param value
* @return NO or MAYBE
*/
public ContainsResult contains(BytesRef value) {
@@ -182,7 +180,7 @@ public class FuzzySet {
* returned by {@link FixedBitSet#getBits}</li>
* </ul>
* @param out Data output stream
- * @throws IOException
+ * @throws IOException If there is a low-level I/O error
*/
public void serialize(DataOutput out) throws IOException
{
@@ -229,7 +227,7 @@ public class FuzzySet {
* Records a value in the set. The referenced bytes are hashed and then modulo n'd where n is the
* chosen size of the internal bitset.
* @param value the key value to be hashed
- * @throws IOException
+ * @throws IOException If there is a low-level I/O error
*/
public void addValue(BytesRef value) throws IOException {
int hash = hashFunction.hash(value);
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java Thu Oct 25 13:10:25 2012
@@ -27,7 +27,7 @@ import java.util.TreeMap;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; // javadocs
+import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat; // javadocs
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo.IndexOptions;
@@ -52,7 +52,7 @@ import org.apache.lucene.util.automaton.
// - build depth-N prefix hash?
// - or: longer dense skip lists than just next byte?
-/** Wraps {@link Lucene40PostingsFormat} format for on-disk
+/** Wraps {@link Lucene41PostingsFormat} format for on-disk
* storage, but then at read time loads and stores all
* terms & postings directly in RAM as byte[], int[].
*
@@ -100,12 +100,12 @@ public final class DirectPostingsFormat
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- return PostingsFormat.forName("Lucene40").fieldsConsumer(state);
+ return PostingsFormat.forName("Lucene41").fieldsConsumer(state);
}
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- FieldsProducer postings = PostingsFormat.forName("Lucene40").fieldsProducer(state);
+ FieldsProducer postings = PostingsFormat.forName("Lucene41").fieldsProducer(state);
if (state.context.context != IOContext.Context.MERGE) {
FieldsProducer loadedPostings;
try {
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java Thu Oct 25 13:10:25 2012
@@ -37,6 +37,7 @@ import org.apache.lucene.store.IndexInpu
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
/** Concrete class that reads the current doc/freq/skip
* postings format.
@@ -101,30 +102,7 @@ public class SepPostingsReader extends P
@Override
public void close() throws IOException {
- try {
- if (freqIn != null)
- freqIn.close();
- } finally {
- try {
- if (docIn != null)
- docIn.close();
- } finally {
- try {
- if (skipIn != null)
- skipIn.close();
- } finally {
- try {
- if (posIn != null) {
- posIn.close();
- }
- } finally {
- if (payloadIn != null) {
- payloadIn.close();
- }
- }
- }
- }
- }
+ IOUtils.close(freqIn, docIn, skipIn, posIn, payloadIn);
}
private static final class SepTermState extends BlockTermState {
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java Thu Oct 25 13:10:25 2012
@@ -198,6 +198,7 @@ public class SimpleTextDocValuesConsumer
IOUtils.close(output);
} else {
IOUtils.closeWhileHandlingException(output);
+ dir.deleteFile(fileName);
}
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java Thu Oct 25 13:10:25 2012
@@ -52,6 +52,7 @@ public class SimpleTextFieldInfosReader
IndexInput input = directory.openInput(fileName, iocontext);
BytesRef scratch = new BytesRef();
+ boolean success = false;
try {
SimpleTextUtil.readLine(input, scratch);
@@ -127,9 +128,15 @@ public class SimpleTextFieldInfosReader
throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
}
- return new FieldInfos(infos);
+ FieldInfos fieldInfos = new FieldInfos(infos);
+ success = true;
+ return fieldInfos;
} finally {
- input.close();
+ if (success) {
+ input.close();
+ } else {
+ IOUtils.closeWhileHandlingException(input);
+ }
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java Thu Oct 25 13:10:25 2012
@@ -29,6 +29,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
/**
* writes plaintext field infos files
@@ -62,6 +63,7 @@ public class SimpleTextFieldInfosWriter
final String fileName = IndexFileNames.segmentFileName(segmentName, "", FIELD_INFOS_EXTENSION);
IndexOutput out = directory.createOutput(fileName, context);
BytesRef scratch = new BytesRef();
+ boolean success = false;
try {
SimpleTextUtil.write(out, NUMFIELDS);
SimpleTextUtil.write(out, Integer.toString(infos.size()), scratch);
@@ -125,8 +127,13 @@ public class SimpleTextFieldInfosWriter
}
}
}
+ success = true;
} finally {
- out.close();
+ if (success) {
+ out.close();
+ } else {
+ IOUtils.closeWhileHandlingException(out);
+ }
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java Thu Oct 25 13:10:25 2012
@@ -96,10 +96,16 @@ public class SimpleTextPerDocProducer ex
@Override
public void close() throws IOException {
+ boolean success = false;
try {
super.close();
+ success = true;
} finally {
- IOUtils.close(input);
+ if (success) {
+ IOUtils.close(input);
+ } else {
+ IOUtils.closeWhileHandlingException(input);
+ }
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java Thu Oct 25 13:10:25 2012
@@ -128,6 +128,10 @@ public class SimpleTextSegmentInfoWriter
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(output);
+ try {
+ dir.deleteFile(segFileName);
+ } catch (Throwable t) {
+ }
} else {
output.close();
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java Thu Oct 25 13:10:25 2012
@@ -18,7 +18,6 @@ package org.apache.lucene.codecs.simplet
*/
import java.io.IOException;
-import java.util.ArrayList;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.index.FieldInfo;
@@ -46,7 +45,7 @@ import static org.apache.lucene.codecs.s
* @lucene.experimental
*/
public class SimpleTextStoredFieldsReader extends StoredFieldsReader {
- private ArrayList<Long> offsets; /* docid -> offset in .fld file */
+ private long offsets[]; /* docid -> offset in .fld file */
private IndexInput in;
private BytesRef scratch = new BytesRef();
private CharsRef scratchUTF16 = new CharsRef();
@@ -60,14 +59,16 @@ public class SimpleTextStoredFieldsReade
success = true;
} finally {
if (!success) {
- close();
+ try {
+ close();
+ } catch (Throwable t) {} // ensure we throw our original exception
}
}
- readIndex();
+ readIndex(si.getDocCount());
}
// used by clone
- SimpleTextStoredFieldsReader(ArrayList<Long> offsets, IndexInput in, FieldInfos fieldInfos) {
+ SimpleTextStoredFieldsReader(long offsets[], IndexInput in, FieldInfos fieldInfos) {
this.offsets = offsets;
this.in = in;
this.fieldInfos = fieldInfos;
@@ -76,19 +77,22 @@ public class SimpleTextStoredFieldsReade
// we don't actually write a .fdx-like index, instead we read the
// stored fields file in entirety up-front and save the offsets
// so we can seek to the documents later.
- private void readIndex() throws IOException {
- offsets = new ArrayList<Long>();
+ private void readIndex(int size) throws IOException {
+ offsets = new long[size];
+ int upto = 0;
while (!scratch.equals(END)) {
readLine();
if (StringHelper.startsWith(scratch, DOC)) {
- offsets.add(in.getFilePointer());
+ offsets[upto] = in.getFilePointer();
+ upto++;
}
}
+ assert upto == offsets.length;
}
@Override
public void visitDocument(int n, StoredFieldVisitor visitor) throws IOException {
- in.seek(offsets.get(n));
+ in.seek(offsets[n]);
readLine();
assert StringHelper.startsWith(scratch, NUM);
int numFields = parseIntAt(NUM.length);
@@ -139,10 +143,9 @@ public class SimpleTextStoredFieldsReade
if (type == TYPE_STRING) {
visitor.stringField(fieldInfo, new String(scratch.bytes, scratch.offset+VALUE.length, scratch.length-VALUE.length, "UTF-8"));
} else if (type == TYPE_BINARY) {
- // TODO: who owns the bytes?
byte[] copy = new byte[scratch.length-VALUE.length];
System.arraycopy(scratch.bytes, scratch.offset+VALUE.length, copy, 0, copy.length);
- visitor.binaryField(fieldInfo, copy, 0, copy.length);
+ visitor.binaryField(fieldInfo, copy);
} else if (type == TYPE_INT) {
UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+VALUE.length, scratch.length-VALUE.length, scratchUTF16);
visitor.intField(fieldInfo, Integer.parseInt(scratchUTF16.toString()));
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java Thu Oct 25 13:10:25 2012
@@ -159,7 +159,7 @@ public class SimpleTextStoredFieldsWrite
public void abort() {
try {
close();
- } catch (IOException ignored) {}
+ } catch (Throwable ignored) {}
IOUtils.deleteFilesIgnoringExceptions(directory, IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION));
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java Thu Oct 25 13:10:25 2012
@@ -18,7 +18,6 @@ package org.apache.lucene.codecs.simplet
*/
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
@@ -54,7 +53,7 @@ import static org.apache.lucene.codecs.s
* @lucene.experimental
*/
public class SimpleTextTermVectorsReader extends TermVectorsReader {
- private ArrayList<Long> offsets; /* docid -> offset in .vec file */
+ private long offsets[]; /* docid -> offset in .vec file */
private IndexInput in;
private BytesRef scratch = new BytesRef();
private CharsRef scratchUTF16 = new CharsRef();
@@ -66,14 +65,16 @@ public class SimpleTextTermVectorsReader
success = true;
} finally {
if (!success) {
- close();
+ try {
+ close();
+ } catch (Throwable t) {} // ensure we throw our original exception
}
}
- readIndex();
+ readIndex(si.getDocCount());
}
// used by clone
- SimpleTextTermVectorsReader(ArrayList<Long> offsets, IndexInput in) {
+ SimpleTextTermVectorsReader(long offsets[], IndexInput in) {
this.offsets = offsets;
this.in = in;
}
@@ -81,26 +82,29 @@ public class SimpleTextTermVectorsReader
// we don't actually write a .tvx-like index, instead we read the
// vectors file in entirety up-front and save the offsets
// so we can seek to the data later.
- private void readIndex() throws IOException {
- offsets = new ArrayList<Long>();
+ private void readIndex(int maxDoc) throws IOException {
+ offsets = new long[maxDoc];
+ int upto = 0;
while (!scratch.equals(END)) {
readLine();
if (StringHelper.startsWith(scratch, DOC)) {
- offsets.add(in.getFilePointer());
+ offsets[upto] = in.getFilePointer();
+ upto++;
}
}
+ assert upto == offsets.length;
}
@Override
public Fields get(int doc) throws IOException {
// TestTV tests for this in testBadParams... but is this
// really guaranteed by the API?
- if (doc < 0 || doc >= offsets.size()) {
+ if (doc < 0 || doc >= offsets.length) {
throw new IllegalArgumentException("doc id out of range");
}
SortedMap<String,SimpleTVTerms> fields = new TreeMap<String,SimpleTVTerms>();
- in.seek(offsets.get(doc));
+ in.seek(offsets[doc]);
readLine();
assert StringHelper.startsWith(scratch, NUMFIELDS);
int numFields = parseIntAt(NUMFIELDS.length);
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java Thu Oct 25 13:10:25 2012
@@ -167,7 +167,7 @@ public class SimpleTextTermVectorsWriter
public void abort() {
try {
close();
- } catch (IOException ignored) {}
+ } catch (Throwable ignored) {}
IOUtils.deleteFilesIgnoringExceptions(directory, IndexFileNames.segmentFileName(segment, "", VECTORS_EXTENSION));
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec Thu Oct 25 13:10:25 2012
@@ -14,4 +14,3 @@
# limitations under the License.
org.apache.lucene.codecs.simpletext.SimpleTextCodec
-org.apache.lucene.codecs.appending.AppendingCodec
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Thu Oct 25 13:10:25 2012
@@ -13,9 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat
+org.apache.lucene.codecs.pulsing.Pulsing41PostingsFormat
org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat
org.apache.lucene.codecs.memory.MemoryPostingsFormat
org.apache.lucene.codecs.bloom.BloomFilteringPostingsFormat
org.apache.lucene.codecs.memory.DirectPostingsFormat
-org.apache.lucene.codecs.block.BlockPostingsFormat
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java Thu Oct 25 13:10:25 2012
@@ -25,7 +25,6 @@ import java.util.Locale;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
@@ -52,7 +51,7 @@ import org.apache.lucene.util._TestUtil;
public class Test10KPulsings extends LuceneTestCase {
public void test10kPulsed() throws Exception {
// we always run this test with pulsing codec.
- Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1));
+ Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing41PostingsFormat(1));
File f = _TestUtil.getTempDir("10kpulsed");
BaseDirectoryWrapper dir = newFSDirectory(f);
@@ -103,7 +102,7 @@ public class Test10KPulsings extends Luc
public void test10kNotPulsed() throws Exception {
// we always run this test with pulsing codec.
int freqCutoff = _TestUtil.nextInt(random(), 1, 10);
- Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(freqCutoff));
+ Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing41PostingsFormat(freqCutoff));
File f = _TestUtil.getTempDir("10knotpulsed");
BaseDirectoryWrapper dir = newFSDirectory(f);
Modified: lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/codecs/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java Thu Oct 25 13:10:25 2012
@@ -45,7 +45,7 @@ public class TestPulsingReuse extends Lu
// TODO: this is a basic test. this thing is complicated, add more
public void testSophisticatedReuse() throws Exception {
// we always run this test with pulsing codec.
- Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1));
+ Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing41PostingsFormat(1));
Directory dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(cp));
@@ -85,7 +85,6 @@ public class TestPulsingReuse extends Lu
// we always run this test with pulsing codec.
Codec cp = _TestUtil.alwaysPostingsFormat(new NestedPulsingPostingsFormat());
BaseDirectoryWrapper dir = newDirectory();
- dir.setCheckIndexOnClose(false); // will do this ourselves, custom codec
RandomIndexWriter iw = new RandomIndexWriter(random(), dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(cp));
Document doc = new Document();
@@ -119,8 +118,6 @@ public class TestPulsingReuse extends Lu
assertEquals(4, allEnums.size());
ir.close();
- CheckIndex ci = new CheckIndex(dir);
- ci.checkIndex(null);
dir.close();
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/common-build.xml?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/common-build.xml (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/common-build.xml Thu Oct 25 13:10:25 2012
@@ -109,6 +109,8 @@
</condition>
<property name="tests.clover.args" value=""/>
+ <property name="tests.heapdump.args" value=""/>
+
<property name="tests.tempDir" location="${build.dir}/test"/>
<property name="tests.cachefile" location="${common.dir}/tools/junit4/cached-timehints.txt" />
@@ -169,6 +171,7 @@
<property name="src.dir" location="src/java"/>
<property name="tests.src.dir" location="src/test"/>
+ <available property="module.has.tests" type="dir" file="${tests.src.dir}"/>
<property name="build.dir" location="build"/>
<!-- Needed in case a module needs the original build, also for compile-tools to be called from a module -->
<property name="common.build.dir" location="${common.dir}/build"/>
@@ -208,6 +211,7 @@
-->
<property name="svnversion.exe" value="svnversion" />
<property name="svn.exe" value="svn" />
+ <property name="perl.exe" value="perl" />
<property name="hg.exe" value="hg" />
<property name="moman.url" value="https://bitbucket.org/jpbarrette/moman" />
@@ -240,6 +244,7 @@
<propertyref regex=".*\.uptodate$$"/>
<propertyref regex=".*\.compiled$$"/>
<propertyref regex=".*\.loaded$$"/>
+ <propertyref name="lucene.javadoc.url"/><!-- for Solr -->
</propertyset>
<patternset id="lucene.local.src.package.patterns"
@@ -260,6 +265,24 @@
</condition>
</fail>
+ <condition property="documentation-lint.supported">
+ <and>
+ <or>
+ <contains string="${java.vm.name}" substring="hotspot" casesensitive="false"/>
+ <contains string="${java.vm.name}" substring="openjdk" casesensitive="false"/>
+ <contains string="${java.vm.name}" substring="jrockit" casesensitive="false"/>
+ </or>
+ <or>
+ <equals arg1="${ant.java.version}" arg2="1.7"/>
+ <equals arg1="${ant.java.version}" arg2="1.8"/>
+ </or>
+ </and>
+ </condition>
+
+ <target name="-documentation-lint-unsupported" unless="documentation-lint.supported">
+ <echo level="warning" message="WARN: Linting documentation HTML is not supported on this Java version (${ant.java.version}) / JVM (${java.vm.name}). NOTHING DONE!"/>
+ </target>
+
<!-- Import custom ANT tasks. -->
<import file="${common.dir}/tools/custom-tasks.xml" />
@@ -768,6 +791,7 @@
<property name="tests.method" value="" />
<property name="tests.dynamicAssignmentRatio" value="0.50" /> <!-- 50% of suites -->
<property name="tests.haltonfailure" value="true" />
+ <property name="tests.leaveTemporary" value="false" />
<property name="tests.iters" value="" />
<property name="tests.dups" value="1" />
@@ -805,7 +829,7 @@
dynamicAssignmentRatio="${tests.dynamicAssignmentRatio}"
shuffleOnSlave="true"
- leaveTemporary="false"
+ leaveTemporary="${tests.leaveTemporary}"
seed="${tests.seed}"
heartbeat="${tests.heartbeat}"
@@ -823,6 +847,7 @@
<!-- JVM arguments and system properties. -->
<jvmarg line="${args}"/>
+ <jvmarg line="${tests.heapdump.args}"/>
<jvmarg line="${tests.clover.args}"/>
<!-- set the number of times tests should run -->
@@ -1113,6 +1138,10 @@ ant -Dtests.timestamps=on ...
# Override forked JVM file.encoding
ant -Dtests.file.encoding=XXX ...
+# Don't remove temporary files under slave directories, even if
+# the test passes.
+ant -Dtests.leaveTemporary=true
+
# Output test files and reports.
${tests-output}/tests-report.txt - full ASCII tests report
${tests-output}/tests-failures.txt - failures only (if any)
@@ -1135,7 +1164,7 @@ ${tests-output}/junit4-*.suites - pe
<!-- note: order here is important, the taskdef depends on test-framework
jars so we just order it after compile-test to ensure that -->
- <target name="test" depends="compile-test,install-junit4-taskdef,clover,validate" description="Runs unit tests">
+ <target name="test" depends="clover,compile-test,install-junit4-taskdef,validate" description="Runs unit tests">
<mkdir dir="${junit.output.dir}"/>
<test-macro threadNum="${tests.jvms}" />
</target>
@@ -1143,9 +1172,9 @@ ${tests-output}/junit4-*.suites - pe
<!--
See http://issues.apache.org/jira/browse/LUCENE-721
-->
- <target name="clover" depends="-clover.disable,-clover.setup" description="Instrument the Unit tests using Clover. To use, specify -Drun.clover=true on the command line."/>
+ <target name="clover" depends="-clover.disable,-clover.load,-clover.classpath,-clover.setup" description="Instrument the Unit tests using Clover. To use, specify -Drun.clover=true on the command line."/>
- <target name="-clover.setup" if="run.clover" unless="clover.loaded">
+ <target name="-clover.load" if="run.clover" unless="clover.loaded">
<available file="${clover.license.path}" property="clover.license.available" />
<fail unless="clover.license.available"><" flags="gs"/>
- <scriptfilter language="javascript" classpathref="pegdown.classpath"><![CDATA[
- importClass(java.lang.StringBuilder);
- importClass(org.pegdown.PegDownProcessor);
- importClass(org.pegdown.Extensions);
- importClass(org.pegdown.FastEncoder);
- var markdownSource = self.getToken();
- var title = undefined;
- // match the first heading in markdown and use as title:
- if (markdownSource.search(/^#+\s*(.+)$/m) >= 0) {
- title = RegExp.$1;
- }
- var processor = new PegDownProcessor(
+ <scriptfilter language="groovy">
+ <classpath>
+ <path refid="groovy.classpath"/>
+ <path refid="pegdown.classpath"/>
+ </classpath><![CDATA[
+ import org.pegdown.PegDownProcessor;
+ import org.pegdown.Extensions;
+ import org.pegdown.FastEncoder;
+
+ String markdownSource = self.getToken();
+ PegDownProcessor processor = new PegDownProcessor(
Extensions.ABBREVIATIONS | Extensions.AUTOLINKS |
Extensions.FENCED_CODE_BLOCKS | Extensions.SMARTS
);
- var html = new StringBuilder('<html>\n<head>\n');
- if (title) {
- html.append('<title>').append(FastEncoder.encode(title)).append('</title>\n');
+ StringBuilder html = new StringBuilder('<html>\n<head>\n');
+ // match the first heading in markdown and use as title:
+ markdownSource.find(~/(?m)^#+\s*(.+)$/) {
+ match, title -> html.append('<title>').append(FastEncoder.encode(title)).append('</title>\n');
}
html.append('<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">\n')
.append('</head>\n<body>\n')
@@ -1829,16 +1946,23 @@ ${tests-output}/junit4-*.suites - pe
compile changes.txt into an html file
-->
<macrodef name="build-changes">
- <attribute name="changes.src.dir" default="${changes.src.dir}"/>
+ <attribute name="changes.product"/>
+ <attribute name="changes.src.file" default="CHANGES.txt"/>
<attribute name="changes.target.dir" default="${changes.target.dir}"/>
<sequential>
<mkdir dir="@{changes.target.dir}"/>
- <exec executable="perl" input="CHANGES.txt" output="@{changes.target.dir}/Changes.html"
+ <get src="https://issues.apache.org/jira/rest/api/2/project/@{changes.product}"
+ dest="@{changes.target.dir}/jiraVersionList.json" httpusecaches="false"/>
+ <exec executable="${perl.exe}" input="@{changes.src.file}" output="@{changes.target.dir}/Changes.html"
failonerror="true" logError="true">
- <arg value="@{changes.src.dir}/changes2html.pl"/>
+ <arg value="-CSD"/>
+ <arg value="${changes.src.dir}/changes2html.pl"/>
+ <arg value="@{changes.product}"/>
+ <arg value="@{changes.target.dir}/jiraVersionList.json"/>
</exec>
+ <delete file="@{changes.target.dir}/jiraVersionList.json"/>
<copy todir="@{changes.target.dir}">
- <fileset dir="@{changes.src.dir}" includes="*.css"/>
+ <fileset dir="${changes.src.dir}" includes="*.css"/>
</copy>
</sequential>
</macrodef>
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Token.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Token.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Token.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/Token.java Thu Oct 25 13:10:25 2012
@@ -523,7 +523,7 @@ public class Token extends CharTermAttri
/**
* Copy the prototype token's fields into this one. Note: Payloads are shared.
- * @param prototype
+ * @param prototype source Token to copy fields from
*/
public void reinit(Token prototype) {
copyBuffer(prototype.buffer(), 0, prototype.length());
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java Thu Oct 25 13:10:25 2012
@@ -160,7 +160,7 @@ public abstract class TokenStream extend
* differ from the offset of the last token eg in case one or more whitespaces
* followed after the last token, but a WhitespaceTokenizer was used.
*
- * @throws IOException
+ * @throws IOException If an I/O error occurs
*/
public void end() throws IOException {
// do nothing by default
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/package.html?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/package.html (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/analysis/package.html Thu Oct 25 13:10:25 2012
@@ -183,7 +183,7 @@ and proximity searches (though sentence
Version matchVersion = Version.LUCENE_XY; // Substitute desired Lucene version for XY
Analyzer analyzer = new StandardAnalyzer(matchVersion); // or any other analyzer
TokenStream ts = analyzer.tokenStream("myfield", new StringReader("some text goes here"));
- OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+ OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
try {
ts.reset(); // Resets this stream to the beginning. (Required)
@@ -734,7 +734,7 @@ public final class PartOfSpeechAttribute
{@literal @Override}
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
final Tokenizer source = new WhitespaceTokenizer(matchVersion, reader);
- TokenStream result = new LengthFilter(source, 3, Integer.MAX_VALUE);
+ TokenStream result = new LengthFilter(true, source, 3, Integer.MAX_VALUE);
result = new PartOfSpeechTaggingFilter(result);
return new TokenStreamComponents(source, result);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTermState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTermState.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTermState.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTermState.java Thu Oct 25 13:10:25 2012
@@ -36,6 +36,11 @@ public class BlockTermState extends OrdT
/** fp into the terms dict primary file (_X.tim) that holds this term */
public long blockFilePointer;
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected BlockTermState() {
+ }
+
@Override
public void copyFrom(TermState _other) {
assert _other instanceof BlockTermState : "can not copy from " + _other.getClass().getName();
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java Thu Oct 25 13:10:25 2012
@@ -99,12 +99,17 @@ public class BlockTreeTermsReader extend
private final TreeMap<String,FieldReader> fields = new TreeMap<String,FieldReader>();
- // keeps the dirStart offset
- protected long dirOffset;
- protected long indexDirOffset;
+ /** File offset where the directory starts in the terms file. */
+ private long dirOffset;
+
+ /** File offset where the directory starts in the index file. */
+ private long indexDirOffset;
private String segment;
+ private final int version;
+
+ /** Sole constructor. */
public BlockTreeTermsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo info,
PostingsReaderBase postingsReader, IOContext ioContext,
String segmentSuffix, int indexDivisor)
@@ -120,11 +125,14 @@ public class BlockTreeTermsReader extend
IndexInput indexIn = null;
try {
- readHeader(in);
+ version = readHeader(in);
if (indexDivisor != -1) {
indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION),
ioContext);
- readIndexHeader(indexIn);
+ int indexVersion = readIndexHeader(indexIn);
+ if (indexVersion != version) {
+ throw new CorruptIndexException("mixmatched version files: " + in + "=" + version + "," + indexIn + "=" + indexVersion);
+ }
}
// Have PostingsReader init itself
@@ -169,32 +177,48 @@ public class BlockTreeTermsReader extend
throw new CorruptIndexException("duplicate field: " + fieldInfo.name + " (resource=" + in + ")");
}
}
+ if (indexDivisor != -1) {
+ indexIn.close();
+ }
+
success = true;
} finally {
if (!success) {
+ // this.close() will close in:
IOUtils.closeWhileHandlingException(indexIn, this);
- } else if (indexDivisor != -1) {
- indexIn.close();
}
}
}
- protected void readHeader(IndexInput input) throws IOException {
- CodecUtil.checkHeader(input, BlockTreeTermsWriter.TERMS_CODEC_NAME,
+ /** Reads terms file header. */
+ private int readHeader(IndexInput input) throws IOException {
+ int version = CodecUtil.checkHeader(input, BlockTreeTermsWriter.TERMS_CODEC_NAME,
BlockTreeTermsWriter.TERMS_VERSION_START,
BlockTreeTermsWriter.TERMS_VERSION_CURRENT);
- dirOffset = input.readLong();
+ if (version < BlockTreeTermsWriter.TERMS_VERSION_APPEND_ONLY) {
+ dirOffset = input.readLong();
+ }
+ return version;
}
- protected void readIndexHeader(IndexInput input) throws IOException {
- CodecUtil.checkHeader(input, BlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
+ /** Reads index file header. */
+ private int readIndexHeader(IndexInput input) throws IOException {
+ int version = CodecUtil.checkHeader(input, BlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
BlockTreeTermsWriter.TERMS_INDEX_VERSION_START,
BlockTreeTermsWriter.TERMS_INDEX_VERSION_CURRENT);
- indexDirOffset = input.readLong();
+ if (version < BlockTreeTermsWriter.TERMS_INDEX_VERSION_APPEND_ONLY) {
+ indexDirOffset = input.readLong();
+ }
+ return version;
}
-
- protected void seekDir(IndexInput input, long dirOffset)
+
+ /** Seek {@code input} to the directory offset. */
+ private void seekDir(IndexInput input, long dirOffset)
throws IOException {
+ if (version >= BlockTreeTermsWriter.TERMS_INDEX_VERSION_APPEND_ONLY) {
+ input.seek(input.length() - 8);
+ dirOffset = input.readLong();
+ }
input.seek(dirOffset);
}
@@ -251,36 +275,70 @@ public class BlockTreeTermsReader extend
* returned by {@link FieldReader#computeStats()}.
*/
public static class Stats {
+ /** How many nodes in the index FST. */
public int indexNodeCount;
+
+ /** How many arcs in the index FST. */
public int indexArcCount;
+
+ /** Byte size of the index. */
public int indexNumBytes;
+ /** Total number of terms in the field. */
public long totalTermCount;
- public long totalTermBytes;
+ /** Total number of bytes (sum of term lengths) across all terms in the field. */
+ public long totalTermBytes;
+ /** The number of normal (non-floor) blocks in the terms file. */
public int nonFloorBlockCount;
+
+ /** The number of floor blocks (meta-blocks larger than the
+ * allowed {@code maxItemsPerBlock}) in the terms file. */
public int floorBlockCount;
+
+ /** The number of sub-blocks within the floor blocks. */
public int floorSubBlockCount;
+
+ /** The number of "internal" blocks (that have both
+ * terms and sub-blocks). */
public int mixedBlockCount;
+
+ /** The number of "leaf" blocks (blocks that have only
+ * terms). */
public int termsOnlyBlockCount;
+
+ /** The number of "internal" blocks that do not contain
+ * terms (have only sub-blocks). */
public int subBlocksOnlyBlockCount;
+
+ /** Total number of blocks. */
public int totalBlockCount;
+ /** Number of blocks at each prefix depth. */
public int[] blockCountByPrefixLen = new int[10];
private int startBlockCount;
private int endBlockCount;
+
+ /** Total number of bytes used to store term suffixes. */
public long totalBlockSuffixBytes;
+
+ /** Total number of bytes used to store term stats (not
+ * including what the {@link PostingsBaseFormat}
+ * stores. */
public long totalBlockStatsBytes;
- // Postings impl plus the other few vInts stored in
- // the frame:
+ /** Total bytes stored by the {@link PostingsBaseFormat},
+ * plus the other few vInts stored in the frame. */
public long totalBlockOtherBytes;
+ /** Segment name. */
public final String segment;
+
+ /** Field name. */
public final String field;
- public Stats(String segment, String field) {
+ Stats(String segment, String field) {
this.segment = segment;
this.field = field;
}
@@ -386,6 +444,7 @@ public class BlockTreeTermsReader extend
final Outputs<BytesRef> fstOutputs = ByteSequenceOutputs.getSingleton();
final BytesRef NO_OUTPUT = fstOutputs.getNoOutput();
+ /** BlockTree's implementation of {@link Terms}. */
public final class FieldReader extends Terms {
final long numTerms;
final FieldInfo fieldInfo;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java Thu Oct 25 13:10:25 2012
@@ -23,10 +23,12 @@ import java.util.Comparator;
import java.util.List;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.ArrayUtil;
@@ -76,6 +78,97 @@ import org.apache.lucene.util.fst.Util;
* Writes terms dict and index, block-encoding (column
* stride) each term's metadata for each set of terms
* between two index terms.
+ * <p>
+ * Files:
+ * <ul>
+ * <li><tt>.tim</tt>: <a href="#Termdictionary">Term Dictionary</a></li>
+ * <li><tt>.tip</tt>: <a href="#Termindex">Term Index</a></li>
+ * </ul>
+ * <p>
+ * <a name="Termdictionary" id="Termdictionary"></a>
+ * <h3>Term Dictionary</h3>
+ *
+ * <p>The .tim file contains the list of terms in each
+ * field along with per-term statistics (such as docfreq)
+ * and per-term metadata (typically pointers to the postings list
+ * for that term in the inverted index).
+ * </p>
+ *
+ * <p>The .tim is arranged in blocks: with blocks containing
+ * a variable number of entries (by default 25-48), where
+ * each entry is either a term or a reference to a
+ * sub-block.</p>
+ *
+ * <p>NOTE: The term dictionary can plug into different postings implementations:
+ * the postings writer/reader are actually responsible for encoding
+ * and decoding the Postings Metadata and Term Metadata sections.</p>
+ *
+ * <ul>
+ * <!-- TODO: expand on this, its not really correct and doesnt explain sub-blocks etc -->
+ * <li>TermsDict (.tim) --> Header, <i>Postings Metadata</i>, Block<sup>NumBlocks</sup>,
+ * FieldSummary, DirOffset</li>
+ * <li>Block --> SuffixBlock, StatsBlock, MetadataBlock</li>
+ * <li>SuffixBlock --> EntryCount, SuffixLength, Byte<sup>SuffixLength</sup></li>
+ * <li>StatsBlock --> StatsLength, <DocFreq, TotalTermFreq><sup>EntryCount</sup></li>
+ * <li>MetadataBlock --> MetaLength, <<i>Term Metadata</i>><sup>EntryCount</sup></li>
+ * <li>FieldSummary --> NumFields, <FieldNumber, NumTerms, RootCodeLength, Byte<sup>RootCodeLength</sup>,
+ * SumDocFreq, DocCount><sup>NumFields</sup></li>
+ * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>DirOffset --> {@link DataOutput#writeLong Uint64}</li>
+ * <li>EntryCount,SuffixLength,StatsLength,DocFreq,MetaLength,NumFields,
+ * FieldNumber,RootCodeLength,DocCount --> {@link DataOutput#writeVInt VInt}</li>
+ * <li>TotalTermFreq,NumTerms,SumTotalTermFreq,SumDocFreq -->
+ * {@link DataOutput#writeVLong VLong}</li>
+ * </ul>
+ * <p>Notes:</p>
+ * <ul>
+ * <li>Header is a {@link CodecUtil#writeHeader CodecHeader} storing the version information
+ * for the BlockTree implementation.</li>
+ * <li>DirOffset is a pointer to the FieldSummary section.</li>
+ * <li>DocFreq is the count of documents which contain the term.</li>
+ * <li>TotalTermFreq is the total number of occurrences of the term. This is encoded
+ * as the difference between the total number of occurrences and the DocFreq.</li>
+ * <li>FieldNumber is the fields number from {@link FieldInfos}. (.fnm)</li>
+ * <li>NumTerms is the number of unique terms for the field.</li>
+ * <li>RootCode points to the root block for the field.</li>
+ * <li>SumDocFreq is the total number of postings, the number of term-document pairs across
+ * the entire field.</li>
+ * <li>DocCount is the number of documents that have at least one posting for this field.</li>
+ * <li>PostingsMetadata and TermMetadata are plugged into by the specific postings implementation:
+ * these contain arbitrary per-file data (such as parameters or versioning information)
+ * and per-term data (such as pointers to inverted files).
+ * </ul>
+ * <a name="Termindex" id="Termindex"></a>
+ * <h3>Term Index</h3>
+ * <p>The .tip file contains an index into the term dictionary, so that it can be
+ * accessed randomly. The index is also used to determine
+ * when a given term cannot exist on disk (in the .tim file), saving a disk seek.</p>
+ * <ul>
+ * <li>TermsIndex (.tip) --> Header, FSTIndex<sup>NumFields</sup>
+ * <IndexStartFP><sup>NumFields</sup>, DirOffset</li>
+ * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>DirOffset --> {@link DataOutput#writeLong Uint64}</li>
+ * <li>IndexStartFP --> {@link DataOutput#writeVLong VLong}</li>
+ * <!-- TODO: better describe FST output here -->
+ * <li>FSTIndex --> {@link FST FST<byte[]>}</li>
+ * </ul>
+ * <p>Notes:</p>
+ * <ul>
+ * <li>The .tip file contains a separate FST for each
+ * field. The FST maps a term prefix to the on-disk
+ * block that holds all terms starting with that
+ * prefix. Each field's IndexStartFP points to its
+ * FST.</li>
+ * <li>DirOffset is a pointer to the start of the IndexStartFPs
+ * for all fields</li>
+ * <li>It's possible that an on-disk block would contain
+ * too many terms (more than the allowed maximum
+ * (default: 48)). When this happens, the block is
+ * sub-divided into new blocks (called "floor
+ * blocks"), and then the output in the FST for the
+ * block's prefix encodes the leading byte of each
+ * sub-block, and its file pointer.
+ * </ul>
*
* @see BlockTreeTermsReader
* @lucene.experimental
@@ -83,11 +176,18 @@ import org.apache.lucene.util.fst.Util;
public class BlockTreeTermsWriter extends FieldsConsumer {
+ /** Suggested default value for the {@code
+ * minItemsInBlock} parameter to {@link
+ * #BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)}. */
public final static int DEFAULT_MIN_BLOCK_SIZE = 25;
+
+ /** Suggested default value for the {@code
+ * maxItemsInBlock} parameter to {@link
+ * #BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)}. */
public final static int DEFAULT_MAX_BLOCK_SIZE = 48;
//public final static boolean DEBUG = false;
- public final static boolean SAVE_DOT_FILES = false;
+ private final static boolean SAVE_DOT_FILES = false;
static final int OUTPUT_FLAGS_NUM_BITS = 2;
static final int OUTPUT_FLAGS_MASK = 0x3;
@@ -97,16 +197,28 @@ public class BlockTreeTermsWriter extend
/** Extension of terms file */
static final String TERMS_EXTENSION = "tim";
final static String TERMS_CODEC_NAME = "BLOCK_TREE_TERMS_DICT";
- // Initial format
+
+ /** Initial terms format. */
public static final int TERMS_VERSION_START = 0;
- public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_START;
+
+ /** Append-only */
+ public static final int TERMS_VERSION_APPEND_ONLY = 1;
+
+ /** Current terms format. */
+ public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_APPEND_ONLY;
/** Extension of terms index file */
static final String TERMS_INDEX_EXTENSION = "tip";
final static String TERMS_INDEX_CODEC_NAME = "BLOCK_TREE_TERMS_INDEX";
- // Initial format
+
+ /** Initial index format. */
public static final int TERMS_INDEX_VERSION_START = 0;
- public static final int TERMS_INDEX_VERSION_CURRENT = TERMS_INDEX_VERSION_START;
+
+ /** Append-only */
+ public static final int TERMS_INDEX_VERSION_APPEND_ONLY = 1;
+
+ /** Current index format. */
+ public static final int TERMS_INDEX_VERSION_CURRENT = TERMS_INDEX_VERSION_APPEND_ONLY;
private final IndexOutput out;
private final IndexOutput indexOut;
@@ -116,7 +228,30 @@ public class BlockTreeTermsWriter extend
final PostingsWriterBase postingsWriter;
final FieldInfos fieldInfos;
FieldInfo currentField;
- private final List<TermsWriter> fields = new ArrayList<TermsWriter>();
+
+ private static class FieldMetaData {
+ public final FieldInfo fieldInfo;
+ public final BytesRef rootCode;
+ public final long numTerms;
+ public final long indexStartFP;
+ public final long sumTotalTermFreq;
+ public final long sumDocFreq;
+ public final int docCount;
+
+ public FieldMetaData(FieldInfo fieldInfo, BytesRef rootCode, long numTerms, long indexStartFP, long sumTotalTermFreq, long sumDocFreq, int docCount) {
+ assert numTerms > 0;
+ this.fieldInfo = fieldInfo;
+ assert rootCode != null: "field=" + fieldInfo.name + " numTerms=" + numTerms;
+ this.rootCode = rootCode;
+ this.indexStartFP = indexStartFP;
+ this.numTerms = numTerms;
+ this.sumTotalTermFreq = sumTotalTermFreq;
+ this.sumDocFreq = sumDocFreq;
+ this.docCount = docCount;
+ }
+ }
+
+ private final List<FieldMetaData> fields = new ArrayList<FieldMetaData>();
// private final String segment;
/** Create a new writer. The number of items (terms or
@@ -174,24 +309,24 @@ public class BlockTreeTermsWriter extend
}
this.indexOut = indexOut;
}
-
- protected void writeHeader(IndexOutput out) throws IOException {
- CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
- out.writeLong(0); // leave space for end index pointer
+
+ /** Writes the terms file header. */
+ private void writeHeader(IndexOutput out) throws IOException {
+ CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
}
- protected void writeIndexHeader(IndexOutput out) throws IOException {
+ /** Writes the index file header. */
+ private void writeIndexHeader(IndexOutput out) throws IOException {
CodecUtil.writeHeader(out, TERMS_INDEX_CODEC_NAME, TERMS_INDEX_VERSION_CURRENT);
- out.writeLong(0); // leave space for end index pointer
}
- protected void writeTrailer(IndexOutput out, long dirStart) throws IOException {
- out.seek(CodecUtil.headerLength(TERMS_CODEC_NAME));
+ /** Writes the terms file trailer. */
+ private void writeTrailer(IndexOutput out, long dirStart) throws IOException {
out.writeLong(dirStart);
}
- protected void writeIndexTrailer(IndexOutput indexOut, long dirStart) throws IOException {
- indexOut.seek(CodecUtil.headerLength(TERMS_INDEX_CODEC_NAME));
+ /** Writes the index file trailer. */
+ private void writeIndexTrailer(IndexOutput indexOut, long dirStart) throws IOException {
indexOut.writeLong(dirStart);
}
@@ -201,9 +336,7 @@ public class BlockTreeTermsWriter extend
//if (DEBUG) System.out.println("\nBTTW.addField seg=" + segment + " field=" + field.name);
assert currentField == null || currentField.name.compareTo(field.name) < 0;
currentField = field;
- final TermsWriter terms = new TermsWriter(field);
- fields.add(terms);
- return terms;
+ return new TermsWriter(field);
}
static long encodeOutput(long fp, boolean hasTerms, boolean isFloor) {
@@ -895,6 +1028,14 @@ public class BlockTreeTermsWriter extend
// System.out.println("SAVED to " + dotFileName);
// w.close();
// }
+
+ fields.add(new FieldMetaData(fieldInfo,
+ ((PendingBlock) pending.get(0)).index.getEmptyOutput(),
+ numTerms,
+ indexStartFP,
+ sumTotalTermFreq,
+ sumDocFreq,
+ docCount));
} else {
assert sumTotalTermFreq == 0 || fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY && sumTotalTermFreq == -1;
assert sumDocFreq == 0;
@@ -912,34 +1053,23 @@ public class BlockTreeTermsWriter extend
IOException ioe = null;
try {
- int nonZeroCount = 0;
- for(TermsWriter field : fields) {
- if (field.numTerms > 0) {
- nonZeroCount++;
- }
- }
-
final long dirStart = out.getFilePointer();
final long indexDirStart = indexOut.getFilePointer();
- out.writeVInt(nonZeroCount);
+ out.writeVInt(fields.size());
- for(TermsWriter field : fields) {
- if (field.numTerms > 0) {
- //System.out.println(" field " + field.fieldInfo.name + " " + field.numTerms + " terms");
- out.writeVInt(field.fieldInfo.number);
- out.writeVLong(field.numTerms);
- final BytesRef rootCode = ((PendingBlock) field.pending.get(0)).index.getEmptyOutput();
- assert rootCode != null: "field=" + field.fieldInfo.name + " numTerms=" + field.numTerms;
- out.writeVInt(rootCode.length);
- out.writeBytes(rootCode.bytes, rootCode.offset, rootCode.length);
- if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
- out.writeVLong(field.sumTotalTermFreq);
- }
- out.writeVLong(field.sumDocFreq);
- out.writeVInt(field.docCount);
- indexOut.writeVLong(field.indexStartFP);
+ for(FieldMetaData field : fields) {
+ //System.out.println(" field " + field.fieldInfo.name + " " + field.numTerms + " terms");
+ out.writeVInt(field.fieldInfo.number);
+ out.writeVLong(field.numTerms);
+ out.writeVInt(field.rootCode.length);
+ out.writeBytes(field.rootCode.bytes, field.rootCode.offset, field.rootCode.length);
+ if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
+ out.writeVLong(field.sumTotalTermFreq);
}
+ out.writeVLong(field.sumDocFreq);
+ out.writeVInt(field.docCount);
+ indexOut.writeVLong(field.indexStartFP);
}
writeTrailer(out, dirStart);
writeIndexTrailer(indexOut, indexDirStart);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/Codec.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/Codec.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/Codec.java Thu Oct 25 13:10:25 2012
@@ -30,8 +30,10 @@ import org.apache.lucene.util.NamedSPILo
* written into the index. In order for the segment to be read, the
* name must resolve to your implementation via {@link #forName(String)}.
* This method uses Java's
- * {@link ServiceLoader Service Provider Interface} to resolve codec names.
+ * {@link ServiceLoader Service Provider Interface} (SPI) to resolve codec names.
* <p>
+ * If you implement your own codec, make sure that it has a no-arg constructor
+ * so SPI can load it.
* @see ServiceLoader
*/
public abstract class Codec implements NamedSPILoader.NamedSPI {
@@ -49,7 +51,7 @@ public abstract class Codec implements N
* SPI mechanism (registered in META-INF/ of your jar file, etc).
* @param name must be all ascii alphanumeric, and less than 128 characters in length.
*/
- public Codec(String name) {
+ protected Codec(String name) {
NamedSPILoader.checkServiceName(name);
this.name = name;
}
@@ -86,11 +88,19 @@ public abstract class Codec implements N
/** looks up a codec by name */
public static Codec forName(String name) {
+ if (loader == null) {
+ throw new IllegalStateException("You called Codec.forName() before all Codecs could be initialized. "+
+ "This likely happens if you call it from a Codec's ctor.");
+ }
return loader.lookup(name);
}
/** returns a list of all available codec names */
public static Set<String> availableCodecs() {
+ if (loader == null) {
+ throw new IllegalStateException("You called Codec.availableCodecs() before all Codecs could be initialized. "+
+ "This likely happens if you call it from a Codec's ctor.");
+ }
return loader.availableServices();
}
@@ -109,7 +119,7 @@ public abstract class Codec implements N
loader.reload(classloader);
}
- private static Codec defaultCodec = Codec.forName("Lucene40");
+ private static Codec defaultCodec = Codec.forName("Lucene41");
/** expert: returns the default codec used for newly created
* {@link IndexWriterConfig}s.
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesArraySource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesArraySource.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesArraySource.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesArraySource.java Thu Oct 25 13:10:25 2012
@@ -50,11 +50,14 @@ public abstract class DocValuesArraySour
templates.put(Type.FLOAT_64, new DoubleValues());
TEMPLATES = Collections.unmodifiableMap(templates);
}
-
+
+ /** Returns the {@link DocValuesArraySource} for the given
+ * {@link Type}. */
public static DocValuesArraySource forType(Type type) {
return TEMPLATES.get(type);
}
+ /** Number of bytes to encode each doc value. */
protected final int bytesPerValue;
DocValuesArraySource(int bytesPerValue, Type type) {
@@ -66,9 +69,13 @@ public abstract class DocValuesArraySour
public abstract BytesRef getBytes(int docID, BytesRef ref);
+ /** Creates a {@link DocValuesArraySource} by loading a
+ * previously saved one from an {@link IndexInput}. */
public abstract DocValuesArraySource newFromInput(IndexInput input, int numDocs)
throws IOException;
-
+
+ /** Creates {@link DocValuesArraySource} from a native
+ * array. */
public abstract DocValuesArraySource newFromArray(Object array);
@Override
@@ -76,10 +83,14 @@ public abstract class DocValuesArraySour
return true;
}
+ /** Encode a long value into the provided {@link
+ * BytesRef}. */
public void toBytes(long value, BytesRef bytesRef) {
copyLong(bytesRef, value);
}
+ /** Encode a double value into the provided {@link
+ * BytesRef}. */
public void toBytes(double value, BytesRef bytesRef) {
copyLong(bytesRef, Double.doubleToRawLongBits(value));
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java Thu Oct 25 13:10:25 2012
@@ -51,9 +51,17 @@ import org.apache.lucene.util.BytesRef;
*/
public abstract class DocValuesConsumer {
+ /** Spare {@link BytesRef} that subclasses can reuse. */
protected final BytesRef spare = new BytesRef();
+ /** Returns the {@link Type} of this consumer. */
protected abstract Type getType();
+
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected DocValuesConsumer() {
+ }
+
/**
* Adds the given {@link StorableField} instance to this
* {@link DocValuesConsumer}
@@ -76,7 +84,7 @@ public abstract class DocValuesConsumer
* the total number of documents in this {@link DocValuesConsumer}.
* Must be greater than or equal the last given docID to
* {@link #add(int, StorableField)}.
- * @throws IOException
+ * @throws IOException If an I/O error occurs
*/
public abstract void finish(int docCount) throws IOException;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesFormat.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/DocValuesFormat.java Thu Oct 25 13:10:25 2012
@@ -28,6 +28,10 @@ import org.apache.lucene.index.SegmentRe
* @lucene.experimental
*/
public abstract class DocValuesFormat {
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected DocValuesFormat() {
+ }
/** Consumes (writes) doc values during indexing. */
public abstract PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosFormat.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosFormat.java Thu Oct 25 13:10:25 2012
@@ -26,6 +26,11 @@ import org.apache.lucene.index.FieldInfo
* @lucene.experimental
*/
public abstract class FieldInfosFormat {
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected FieldInfosFormat() {
+ }
+
/** Returns a {@link FieldInfosReader} to read field infos
* from the index */
public abstract FieldInfosReader getFieldInfosReader() throws IOException;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java Thu Oct 25 13:10:25 2012
@@ -28,5 +28,12 @@ import org.apache.lucene.store.IOContext
* @lucene.experimental
*/
public abstract class FieldInfosReader {
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected FieldInfosReader() {
+ }
+
+ /** Read the {@link FieldInfos} previously written with {@link
+ * FieldInfosWriter}. */
public abstract FieldInfos read(Directory directory, String segmentName, IOContext iocontext) throws IOException;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java Thu Oct 25 13:10:25 2012
@@ -28,5 +28,12 @@ import org.apache.lucene.store.IOContext
* @lucene.experimental
*/
public abstract class FieldInfosWriter {
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected FieldInfosWriter() {
+ }
+
+ /** Writes the provided {@link FieldInfos} to the
+ * directory. */
public abstract void write(Directory directory, String segmentName, FieldInfos infos, IOContext context) throws IOException;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java Thu Oct 25 13:10:25 2012
@@ -45,12 +45,22 @@ import org.apache.lucene.index.Terms;
*/
public abstract class FieldsConsumer implements Closeable {
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected FieldsConsumer() {
+ }
+
/** Add a new field */
public abstract TermsConsumer addField(FieldInfo field) throws IOException;
/** Called when we are done adding everything. */
public abstract void close() throws IOException;
+ /** Called during merging to merge all {@link Fields} from
+ * sub-readers. This must recurse to merge all postings
+ * (terms, docs, positions, etc.). A {@link
+ * PostingsFormat} can override this default
+ * implementation to do its own merging. */
public void merge(MergeState mergeState, Fields fields) throws IOException {
for (String field : fields) {
mergeState.fieldInfo = mergeState.fieldInfos.fieldInfo(field);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java Thu Oct 25 13:10:25 2012
@@ -29,5 +29,11 @@ import org.apache.lucene.index.Fields;
*/
public abstract class FieldsProducer extends Fields implements Closeable {
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected FieldsProducer() {
+ }
+
+ @Override
public abstract void close() throws IOException;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java Thu Oct 25 13:10:25 2012
@@ -21,17 +21,13 @@ package org.apache.lucene.codecs;
* A codec that forwards all its method calls to another codec.
* <p>
* Extend this class when you need to reuse the functionality of an existing
- * codec. For example, if you want to build a codec that redefines Lucene40's
+ * codec. For example, if you want to build a codec that redefines Lucene41's
* {@link LiveDocsFormat}:
* <pre class="prettyprint">
* public final class CustomCodec extends FilterCodec {
*
* public CustomCodec() {
- * super("CustomCodec");
- * }
- *
- * public Codec delegate() {
- * return Codec.forName("Lucene40");
+ * super("CustomCodec", new Lucene41Codec());
* }
*
* public LiveDocsFormat liveDocsFormat() {
@@ -40,57 +36,67 @@ package org.apache.lucene.codecs;
*
* }
* </pre>
+ *
+ * <p><em>Please note:</em> Don't call {@link Codec#forName} from
+ * the no-arg constructor of your own codec. When the SPI framework
+ * loads your own Codec as SPI component, SPI has not yet fully initialized!
+ * If you want to extend another Codec, instantiate it directly by calling
+ * its constructor.
+ *
+ * @lucene.experimental
*/
public abstract class FilterCodec extends Codec {
- public FilterCodec(String name) {
+ /** The codec to filter. */
+ protected final Codec delegate;
+
+ /** Sole constructor. When subclassing this codec,
+ * create a no-arg ctor and pass the delegate codec
+ * and a unique name to this ctor.
+ */
+ protected FilterCodec(String name, Codec delegate) {
super(name);
+ this.delegate = delegate;
}
- /**
- * Return the codec that is responsible for providing default format
- * implementations.
- */
- protected abstract Codec delegate();
-
@Override
public DocValuesFormat docValuesFormat() {
- return delegate().docValuesFormat();
+ return delegate.docValuesFormat();
}
@Override
public FieldInfosFormat fieldInfosFormat() {
- return delegate().fieldInfosFormat();
+ return delegate.fieldInfosFormat();
}
@Override
public LiveDocsFormat liveDocsFormat() {
- return delegate().liveDocsFormat();
+ return delegate.liveDocsFormat();
}
@Override
public NormsFormat normsFormat() {
- return delegate().normsFormat();
+ return delegate.normsFormat();
}
@Override
public PostingsFormat postingsFormat() {
- return delegate().postingsFormat();
+ return delegate.postingsFormat();
}
@Override
public SegmentInfoFormat segmentInfoFormat() {
- return delegate().segmentInfoFormat();
+ return delegate.segmentInfoFormat();
}
@Override
public StoredFieldsFormat storedFieldsFormat() {
- return delegate().storedFieldsFormat();
+ return delegate.storedFieldsFormat();
}
@Override
public TermVectorsFormat termVectorsFormat() {
- return delegate().termVectorsFormat();
+ return delegate.termVectorsFormat();
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/LiveDocsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/LiveDocsFormat.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/LiveDocsFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/LiveDocsFormat.java Thu Oct 25 13:10:25 2012
@@ -29,6 +29,12 @@ import org.apache.lucene.util.MutableBit
/** Format for live/deleted documents
* @lucene.experimental */
public abstract class LiveDocsFormat {
+
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected LiveDocsFormat() {
+ }
+
/** Creates a new MutableBits, with all bits set, for the specified size. */
public abstract MutableBits newLiveDocs(int size) throws IOException;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/MappingMultiDocsAndPositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/MappingMultiDocsAndPositionsEnum.java?rev=1402140&r1=1402139&r2=1402140&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/MappingMultiDocsAndPositionsEnum.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/MappingMultiDocsAndPositionsEnum.java Thu Oct 25 13:10:25 2012
@@ -42,6 +42,10 @@ public final class MappingMultiDocsAndPo
int doc = -1;
private MergeState mergeState;
+ /** Sole constructor. */
+ public MappingMultiDocsAndPositionsEnum() {
+ }
+
MappingMultiDocsAndPositionsEnum reset(MultiDocsAndPositionsEnum postingsEnum) {
this.numSubs = postingsEnum.getNumSubs();
this.subs = postingsEnum.getSubs();
@@ -50,14 +54,19 @@ public final class MappingMultiDocsAndPo
return this;
}
+ /** Sets the {@link MergeState}, which is used to re-map
+ * document IDs. */
public void setMergeState(MergeState mergeState) {
this.mergeState = mergeState;
}
+ /** How many sub-readers we are merging.
+ * @see #getSubs */
public int getNumSubs() {
return numSubs;
}
+ /** Returns sub-readers we are merging. */
public EnumWithSlice[] getSubs() {
return subs;
}