You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/11/10 13:54:37 UTC
svn commit: r1200310 - in /lucene/dev/branches/lucene2621/lucene:
contrib/misc/src/test/org/apache/lucene/index/
src/java/org/apache/lucene/index/ src/java/org/apache/lucene/index/codecs/
src/test/org/apache/lucene/index/
Author: rmuir
Date: Thu Nov 10 12:54:36 2011
New Revision: 1200310
URL: http://svn.apache.org/viewvc?rev=1200310&view=rev
Log:
LUCENE-2621: rote move of term vectors reading to codec (step 1)
Added:
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsReader.java
- copied, changed from r1199842, lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/ParallelArrayTermVectorMapper.java (with props)
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentTermPositionVector.java
- copied, changed from r1199842, lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentTermPositionVector.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentTermVector.java
- copied, changed from r1199842, lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentTermVector.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/TermVectorsReader.java (with props)
Removed:
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentTermPositionVector.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentTermVector.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java
Modified:
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/test/org/apache/lucene/index/TestTermVectorAccessor.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
Modified: lucene/dev/branches/lucene2621/lucene/contrib/misc/src/test/org/apache/lucene/index/TestTermVectorAccessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/contrib/misc/src/test/org/apache/lucene/index/TestTermVectorAccessor.java?rev=1200310&r1=1200309&r2=1200310&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/contrib/misc/src/test/org/apache/lucene/index/TestTermVectorAccessor.java (original)
+++ lucene/dev/branches/lucene2621/lucene/contrib/misc/src/test/org/apache/lucene/index/TestTermVectorAccessor.java Thu Nov 10 12:54:36 2011
@@ -4,6 +4,7 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.codecs.ParallelArrayTermVectorMapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1200310&r1=1200309&r2=1200310&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java Thu Nov 10 12:54:36 2011
@@ -25,6 +25,7 @@ import org.apache.lucene.index.codecs.Po
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.StoredFieldsReader;
import org.apache.lucene.index.codecs.PerDocValues;
+import org.apache.lucene.index.codecs.DefaultTermVectorsReader;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -56,7 +57,7 @@ final class SegmentCoreReaders {
private final SegmentReader owner;
StoredFieldsReader fieldsReaderOrig;
- TermVectorsReader termVectorsReaderOrig;
+ DefaultTermVectorsReader termVectorsReaderOrig;
CompoundFileDirectory cfsReader;
CompoundFileDirectory storeCFSReader;
@@ -106,7 +107,7 @@ final class SegmentCoreReaders {
this.owner = owner;
}
- synchronized TermVectorsReader getTermVectorsReaderOrig() {
+ synchronized DefaultTermVectorsReader getTermVectorsReaderOrig() {
return termVectorsReaderOrig;
}
@@ -169,7 +170,8 @@ final class SegmentCoreReaders {
fieldsReaderOrig = si.getCodec().storedFieldsFormat().fieldsReader(storeDir, si, fieldInfos, context);
if (si.getHasVectors()) { // open term vector files only as needed
- termVectorsReaderOrig = new TermVectorsReader(storeDir, storesSegment, fieldInfos, context, si.getDocStoreOffset(), si.docCount);
+ // nocommit
+ termVectorsReaderOrig = new DefaultTermVectorsReader(storeDir, storesSegment, fieldInfos, context, si.getDocStoreOffset(), si.docCount);
}
}
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1200310&r1=1200309&r2=1200310&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Thu Nov 10 12:54:36 2011
@@ -30,6 +30,8 @@ import org.apache.lucene.index.codecs.Co
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.StoredFieldsWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultTermVectorsReader;
+import org.apache.lucene.index.codecs.TermVectorsReader;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -229,6 +231,7 @@ final class SegmentMerger {
* Merge the TermVectors from each of the segments into the new one.
* @throws IOException
*/
+ // nocommit: move to codec
private final void mergeVectors(SegmentWriteState segmentWriteState) throws IOException {
TermVectorsWriter termVectorsWriter = new TermVectorsWriter(directory, segment, mergeState.fieldInfos, context);
// Used for bulk-reading raw bytes for term vectors
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1200310&r1=1200309&r2=1200310&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentReader.java Thu Nov 10 12:54:36 2011
@@ -32,6 +32,7 @@ import org.apache.lucene.store.IndexInpu
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.codecs.StoredFieldsReader;
import org.apache.lucene.index.codecs.PerDocValues;
+import org.apache.lucene.index.codecs.TermVectorsReader;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.Bits;
@@ -667,11 +668,7 @@ public class SegmentReader extends Index
if (orig == null) {
return null;
} else {
- try {
- tvReader = (TermVectorsReader) orig.clone();
- } catch (CloneNotSupportedException cnse) {
- return null;
- }
+ tvReader = orig.clone();
}
termVectorsLocal.set(tvReader);
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=1200310&r1=1200309&r2=1200310&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java Thu Nov 10 12:54:36 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Map;
+import org.apache.lucene.index.codecs.DefaultTermVectorsReader;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
@@ -101,9 +102,9 @@ final class TermVectorsTermsWriter exten
tvd = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION), context);
tvf = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_FIELDS_EXTENSION), context);
- tvx.writeInt(TermVectorsReader.FORMAT_CURRENT);
- tvd.writeInt(TermVectorsReader.FORMAT_CURRENT);
- tvf.writeInt(TermVectorsReader.FORMAT_CURRENT);
+ tvx.writeInt(DefaultTermVectorsReader.FORMAT_CURRENT);
+ tvd.writeInt(DefaultTermVectorsReader.FORMAT_CURRENT);
+ tvf.writeInt(DefaultTermVectorsReader.FORMAT_CURRENT);
success = true;
} finally {
if (!success) {
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java?rev=1200310&r1=1200309&r2=1200310&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java Thu Nov 10 12:54:36 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.index.codecs.DefaultTermVectorsReader;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
@@ -126,9 +127,9 @@ final class TermVectorsTermsWriterPerFie
tvf.writeVInt(numPostings);
byte bits = 0x0;
if (doVectorPositions)
- bits |= TermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR;
+ bits |= DefaultTermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR;
if (doVectorOffsets)
- bits |= TermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR;
+ bits |= DefaultTermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR;
tvf.writeByte(bits);
int lastLen = 0;
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java?rev=1200310&r1=1200309&r2=1200310&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java Thu Nov 10 12:54:36 2011
@@ -17,6 +17,8 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import org.apache.lucene.index.codecs.DefaultTermVectorsReader;
+import org.apache.lucene.index.codecs.TermVectorsReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
@@ -37,11 +39,11 @@ final class TermVectorsWriter {
try {
// Open files for TermVector storage
tvx = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_INDEX_EXTENSION), context);
- tvx.writeInt(TermVectorsReader.FORMAT_CURRENT);
+ tvx.writeInt(DefaultTermVectorsReader.FORMAT_CURRENT);
tvd = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION), context);
- tvd.writeInt(TermVectorsReader.FORMAT_CURRENT);
+ tvd.writeInt(DefaultTermVectorsReader.FORMAT_CURRENT);
tvf = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION), context);
- tvf.writeInt(TermVectorsReader.FORMAT_CURRENT);
+ tvf.writeInt(DefaultTermVectorsReader.FORMAT_CURRENT);
success = true;
} finally {
if (!success) {
@@ -92,8 +94,8 @@ final class TermVectorsWriter {
tpVector = (TermPositionVector) vectors[i];
storePositions = tpVector.size() > 0 && tpVector.getTermPositions(0) != null;
storeOffsets = tpVector.size() > 0 && tpVector.getOffsets(0) != null;
- bits = (byte) ((storePositions ? TermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR : 0) +
- (storeOffsets ? TermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR : 0));
+ bits = (byte) ((storePositions ? DefaultTermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR : 0) +
+ (storeOffsets ? DefaultTermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR : 0));
} else {
tpVector = null;
bits = 0;
Copied: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsReader.java (from r1199842, lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsReader.java?p2=lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsReader.java&p1=lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java&r1=1199842&r2=1200310&rev=1200310&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsReader.java Thu Nov 10 12:54:36 2011
@@ -1,4 +1,4 @@
-package org.apache.lucene.index;
+package org.apache.lucene.index.codecs;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,6 +17,14 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexFormatTooNewException;
+import org.apache.lucene.index.IndexFormatTooOldException;
+import org.apache.lucene.index.TermFreqVector;
+import org.apache.lucene.index.TermVectorMapper;
+import org.apache.lucene.index.TermVectorOffsetInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -24,11 +32,10 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
-import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
-class TermVectorsReader implements Cloneable, Closeable {
+public class DefaultTermVectorsReader extends TermVectorsReader {
// NOTE: if you make a new format, it must be larger than
// the current format
@@ -38,7 +45,8 @@ class TermVectorsReader implements Clone
// NOTE: always change this if you switch to a new format!
// whenever you add a new format, make it 1 larger (positive version logic)!
- static final int FORMAT_CURRENT = FORMAT_UTF8_LENGTH_IN_BYTES;
+ // nocommit: not public
+ public static final int FORMAT_CURRENT = FORMAT_UTF8_LENGTH_IN_BYTES;
// when removing support for old versions, leave the last supported version here
static final int FORMAT_MINIMUM = FORMAT_UTF8_LENGTH_IN_BYTES;
@@ -46,8 +54,10 @@ class TermVectorsReader implements Clone
//The size in bytes that the FORMAT_VERSION will take up at the beginning of each file
static final int FORMAT_SIZE = 4;
- static final byte STORE_POSITIONS_WITH_TERMVECTOR = 0x1;
- static final byte STORE_OFFSET_WITH_TERMVECTOR = 0x2;
+ // nocommit: not public
+ public static final byte STORE_POSITIONS_WITH_TERMVECTOR = 0x1;
+ // nocommit: not public
+ public static final byte STORE_OFFSET_WITH_TERMVECTOR = 0x2;
private FieldInfos fieldInfos;
@@ -63,12 +73,25 @@ class TermVectorsReader implements Clone
private final int format;
- TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos, IOContext context)
+ // only used by a test: nocommit
+ public DefaultTermVectorsReader(Directory d, String segment, FieldInfos fieldInfos, IOContext context)
throws CorruptIndexException, IOException {
this(d, segment, fieldInfos, context, -1, 0);
}
+
+ // used by clone: nocommit
+ DefaultTermVectorsReader(FieldInfos fieldInfos, IndexInput tvx, IndexInput tvd, IndexInput tvf, int size, int numTotalDocs, int docStoreOffset, int format) {
+ this.fieldInfos = fieldInfos;
+ this.tvx = tvx;
+ this.tvd = tvd;
+ this.tvf = tvf;
+ this.size = size;
+ this.numTotalDocs = numTotalDocs;
+ this.docStoreOffset = docStoreOffset;
+ this.format = format;
+ }
- TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos, IOContext context, int docStoreOffset, int size)
+ public DefaultTermVectorsReader(Directory d, String segment, FieldInfos fieldInfos, IOContext context, int docStoreOffset, int size)
throws CorruptIndexException, IOException {
boolean success = false;
@@ -115,12 +138,16 @@ class TermVectorsReader implements Clone
}
// Used for bulk copy when merging
- IndexInput getTvdStream() {
+ // nocommit: not public
+ @Override
+ public IndexInput getTvdStream() {
return tvd;
}
// Used for bulk copy when merging
- IndexInput getTvfStream() {
+ // nocommit: not public
+ @Override
+ public IndexInput getTvfStream() {
return tvf;
}
@@ -128,7 +155,8 @@ class TermVectorsReader implements Clone
tvx.seek((docNum + docStoreOffset) * 16L + FORMAT_SIZE);
}
- boolean canReadRawDocs() {
+ @Override
+ public boolean canReadRawDocs() {
// we can always read raw docs, unless the term vectors
// didn't exist
return format != 0;
@@ -140,7 +168,9 @@ class TermVectorsReader implements Clone
* merging segments, if the field numbers are
* congruent. Once this returns, the tvf & tvd streams
* are seeked to the startDocID. */
- final void rawDocs(int[] tvdLengths, int[] tvfLengths, int startDocID, int numDocs) throws IOException {
+ // nocommit: not public
+ @Override
+ public final void rawDocs(int[] tvdLengths, int[] tvfLengths, int startDocID, int numDocs) throws IOException {
if (tvx == null) {
Arrays.fill(tvdLengths, 0);
@@ -201,6 +231,7 @@ class TermVectorsReader implements Clone
return size;
}
+ @Override
public void get(int docNum, String field, TermVectorMapper mapper) throws IOException {
if (tvx != null) {
int fieldNumber = fieldInfos.fieldNumber(field);
@@ -252,8 +283,9 @@ class TermVectorsReader implements Clone
* @param field The field within the document to retrieve
* @return The TermFreqVector for the document and field or null if there is no termVector for this field.
* @throws IOException if there is an error reading the term vector files
- */
- TermFreqVector get(int docNum, String field) throws IOException {
+ */
+ @Override
+ public TermFreqVector get(int docNum, String field) throws IOException {
// Check if no term vectors are available for this segment at all
ParallelArrayTermVectorMapper mapper = new ParallelArrayTermVectorMapper();
get(docNum, field, mapper);
@@ -299,7 +331,8 @@ class TermVectorsReader implements Clone
* @return All term frequency vectors
* @throws IOException if there is an error reading the term vector files
*/
- TermFreqVector[] get(int docNum) throws IOException {
+ @Override
+ public TermFreqVector[] get(int docNum) throws IOException {
TermFreqVector[] result = null;
if (tvx != null) {
//We need to offset by
@@ -460,79 +493,20 @@ class TermVectorsReader implements Clone
}
@Override
- protected Object clone() throws CloneNotSupportedException {
-
- final TermVectorsReader clone = (TermVectorsReader) super.clone();
+ public TermVectorsReader clone() {
+ IndexInput cloneTvx = null;
+ IndexInput cloneTvd = null;
+ IndexInput cloneTvf = null;
// These are null when a TermVectorsReader was created
// on a segment that did not have term vectors saved
if (tvx != null && tvd != null && tvf != null) {
- clone.tvx = (IndexInput) tvx.clone();
- clone.tvd = (IndexInput) tvd.clone();
- clone.tvf = (IndexInput) tvf.clone();
+ cloneTvx = (IndexInput) tvx.clone();
+ cloneTvd = (IndexInput) tvd.clone();
+ cloneTvf = (IndexInput) tvf.clone();
}
- return clone;
+ return new DefaultTermVectorsReader(fieldInfos, cloneTvx, cloneTvd, cloneTvf, size, numTotalDocs, docStoreOffset, format);
}
}
-
-/**
- * Models the existing parallel array structure
- */
-class ParallelArrayTermVectorMapper extends TermVectorMapper
-{
-
- private BytesRef[] terms;
- private int[] termFreqs;
- private int positions[][];
- private TermVectorOffsetInfo offsets[][];
- private int currentPosition;
- private boolean storingOffsets;
- private boolean storingPositions;
- private String field;
-
- @Override
- public void setExpectations(String field, int numTerms, boolean storeOffsets, boolean storePositions) {
- this.field = field;
- terms = new BytesRef[numTerms];
- termFreqs = new int[numTerms];
- this.storingOffsets = storeOffsets;
- this.storingPositions = storePositions;
- if(storePositions)
- this.positions = new int[numTerms][];
- if(storeOffsets)
- this.offsets = new TermVectorOffsetInfo[numTerms][];
- }
-
- @Override
- public void map(BytesRef term, int frequency, TermVectorOffsetInfo[] offsets, int[] positions) {
- terms[currentPosition] = term;
- termFreqs[currentPosition] = frequency;
- if (storingOffsets)
- {
- this.offsets[currentPosition] = offsets;
- }
- if (storingPositions)
- {
- this.positions[currentPosition] = positions;
- }
- currentPosition++;
- }
-
- /**
- * Construct the vector
- * @return The {@link TermFreqVector} based on the mappings.
- */
- public TermFreqVector materializeVector() {
- SegmentTermVector tv = null;
- if (field != null && terms != null) {
- if (storingPositions || storingOffsets) {
- tv = new SegmentTermPositionVector(field, terms, termFreqs, positions, offsets);
- } else {
- tv = new SegmentTermVector(field, terms, termFreqs);
- }
- }
- return tv;
- }
-}
Added: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/ParallelArrayTermVectorMapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/ParallelArrayTermVectorMapper.java?rev=1200310&view=auto
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/ParallelArrayTermVectorMapper.java (added)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/ParallelArrayTermVectorMapper.java Thu Nov 10 12:54:36 2011
@@ -0,0 +1,83 @@
+package org.apache.lucene.index.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.TermFreqVector;
+import org.apache.lucene.index.TermVectorMapper;
+import org.apache.lucene.index.TermVectorOffsetInfo;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Models the existing parallel array structure
+ */
+// nocommit: not public? wtf is this?
+public class ParallelArrayTermVectorMapper extends TermVectorMapper {
+
+ private BytesRef[] terms;
+ private int[] termFreqs;
+ private int positions[][];
+ private TermVectorOffsetInfo offsets[][];
+ private int currentPosition;
+ private boolean storingOffsets;
+ private boolean storingPositions;
+ private String field;
+
+ @Override
+ public void setExpectations(String field, int numTerms, boolean storeOffsets, boolean storePositions) {
+ this.field = field;
+ terms = new BytesRef[numTerms];
+ termFreqs = new int[numTerms];
+ this.storingOffsets = storeOffsets;
+ this.storingPositions = storePositions;
+ if(storePositions)
+ this.positions = new int[numTerms][];
+ if(storeOffsets)
+ this.offsets = new TermVectorOffsetInfo[numTerms][];
+ }
+
+ @Override
+ public void map(BytesRef term, int frequency, TermVectorOffsetInfo[] offsets, int[] positions) {
+ terms[currentPosition] = term;
+ termFreqs[currentPosition] = frequency;
+ if (storingOffsets)
+ {
+ this.offsets[currentPosition] = offsets;
+ }
+ if (storingPositions)
+ {
+ this.positions[currentPosition] = positions;
+ }
+ currentPosition++;
+ }
+
+ /**
+ * Construct the vector
+ * @return The {@link TermFreqVector} based on the mappings.
+ */
+ public TermFreqVector materializeVector() {
+ SegmentTermVector tv = null;
+ if (field != null && terms != null) {
+ if (storingPositions || storingOffsets) {
+ tv = new SegmentTermPositionVector(field, terms, termFreqs, positions, offsets);
+ } else {
+ tv = new SegmentTermVector(field, terms, termFreqs);
+ }
+ }
+ return tv;
+ }
+}
\ No newline at end of file
Copied: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentTermPositionVector.java (from r1199842, lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentTermPositionVector.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentTermPositionVector.java?p2=lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentTermPositionVector.java&p1=lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentTermPositionVector.java&r1=1199842&r2=1200310&rev=1200310&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentTermPositionVector.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentTermPositionVector.java Thu Nov 10 12:54:36 2011
@@ -1,5 +1,7 @@
-package org.apache.lucene.index;
+package org.apache.lucene.index.codecs;
+import org.apache.lucene.index.TermPositionVector;
+import org.apache.lucene.index.TermVectorOffsetInfo;
import org.apache.lucene.util.BytesRef;
/**
Copied: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentTermVector.java (from r1199842, lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentTermVector.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentTermVector.java?p2=lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentTermVector.java&p1=lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentTermVector.java&r1=1199842&r2=1200310&rev=1200310&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentTermVector.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentTermVector.java Thu Nov 10 12:54:36 2011
@@ -1,4 +1,4 @@
-package org.apache.lucene.index;
+package org.apache.lucene.index.codecs;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.util.*;
+import org.apache.lucene.index.TermFreqVector;
import org.apache.lucene.util.BytesRef;
Added: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/TermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/TermVectorsReader.java?rev=1200310&view=auto
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/TermVectorsReader.java (added)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/TermVectorsReader.java Thu Nov 10 12:54:36 2011
@@ -0,0 +1,42 @@
+package org.apache.lucene.index.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.lucene.index.TermFreqVector;
+import org.apache.lucene.index.TermVectorMapper;
+import org.apache.lucene.store.IndexInput;
+
+public abstract class TermVectorsReader implements Cloneable,Closeable {
+ // TODO: can we consolidate all these get's?
+ public abstract TermFreqVector[] get(int doc) throws IOException;
+ public abstract TermFreqVector get(int doc, String field) throws IOException;
+
+ public abstract void get(int doc, TermVectorMapper mapper) throws IOException;
+ public abstract void get(int doc, String field, TermVectorMapper mapper) throws IOException;
+
+ public abstract TermVectorsReader clone();
+
+ // nocommit: nuke all of these methods below:
+ public abstract boolean canReadRawDocs();
+ public abstract void rawDocs(int[] tvdLengths, int[] tvfLengths, int startDocID, int numDocs) throws IOException;
+ public abstract IndexInput getTvdStream();
+ public abstract IndexInput getTvfStream();
+}
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=1200310&r1=1200309&r2=1200310&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java Thu Nov 10 12:54:36 2011
@@ -32,6 +32,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.codecs.DefaultTermVectorsReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -189,7 +190,7 @@ public class TestTermVectorsReader exten
}
public void testReader() throws IOException {
- TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
+ DefaultTermVectorsReader reader = new DefaultTermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
for (int j = 0; j < 5; j++) {
TermFreqVector vector = reader.get(j, testFields[0]);
assertTrue(vector != null);
@@ -206,7 +207,7 @@ public class TestTermVectorsReader exten
}
public void testPositionReader() throws IOException {
- TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
+ DefaultTermVectorsReader reader = new DefaultTermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
TermPositionVector vector;
BytesRef[] terms;
vector = (TermPositionVector) reader.get(0, testFields[0]);
@@ -249,7 +250,7 @@ public class TestTermVectorsReader exten
}
public void testOffsetReader() throws IOException {
- TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
+ DefaultTermVectorsReader reader = new DefaultTermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
TermPositionVector vector = (TermPositionVector) reader.get(0, testFields[0]);
assertTrue(vector != null);
BytesRef[] terms = vector.getTerms();
@@ -278,7 +279,7 @@ public class TestTermVectorsReader exten
}
public void testMapper() throws IOException {
- TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
+ DefaultTermVectorsReader reader = new DefaultTermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
SortedTermVectorMapper mapper = new SortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());
reader.get(0, mapper);
SortedSet<TermVectorEntry> set = mapper.getTermVectorEntrySet();
@@ -397,9 +398,9 @@ public class TestTermVectorsReader exten
* Make sure exceptions and bad params are handled appropriately
*/
public void testBadParams() throws IOException {
- TermVectorsReader reader = null;
+ DefaultTermVectorsReader reader = null;
try {
- reader = new TermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
+ reader = new DefaultTermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
//Bad document number, good field number
reader.get(50, testFields[0]);
fail();
@@ -409,7 +410,7 @@ public class TestTermVectorsReader exten
reader.close();
}
try {
- reader = new TermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
+ reader = new DefaultTermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
//Bad document number, no field
reader.get(50);
fail();
@@ -419,7 +420,7 @@ public class TestTermVectorsReader exten
reader.close();
}
try {
- reader = new TermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
+ reader = new DefaultTermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
//good document number, bad field number
TermFreqVector vector = reader.get(0, "f50");
assertTrue(vector == null);