You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/11/11 04:38:28 UTC
svn commit: r1200724 - in
/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs:
DefaultTermVectorsWriter.java TermVectorsWriter.java
Author: rmuir
Date: Fri Nov 11 03:38:27 2011
New Revision: 1200724
URL: http://svn.apache.org/viewvc?rev=1200724&view=rev
Log:
LUCENE-2621: default termvectors merge implementation
Modified:
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsWriter.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/TermVectorsWriter.java
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsWriter.java?rev=1200724&r1=1200723&r2=1200724&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsWriter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultTermVectorsWriter.java Fri Nov 11 03:38:27 2011
@@ -18,14 +18,11 @@ package org.apache.lucene.index.codecs;
*/
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.MergePolicy.MergeAbortedException;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.TermFreqVector;
-import org.apache.lucene.index.TermPositionVector;
-import org.apache.lucene.index.TermVectorOffsetInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
@@ -61,124 +58,7 @@ public final class DefaultTermVectorsWri
}
}
}
-
- /**
- * Add a complete document specified by all its term vectors. If document has no
- * term vectors, add value for tvx.
- *
- * @param vectors
- * @throws IOException
- */
- @Override
- public final void addAllDocVectors(TermFreqVector[] vectors, FieldInfos fieldInfos) throws IOException {
-
- tvx.writeLong(tvd.getFilePointer());
- tvx.writeLong(tvf.getFilePointer());
-
- if (vectors != null) {
- final int numFields = vectors.length;
- tvd.writeVInt(numFields);
-
- long[] fieldPointers = new long[numFields];
-
- for (int i=0; i<numFields; i++) {
- fieldPointers[i] = tvf.getFilePointer();
-
- final int fieldNumber = fieldInfos.fieldNumber(vectors[i].getField());
-
- // 1st pass: write field numbers to tvd
- tvd.writeVInt(fieldNumber);
-
- final int numTerms = vectors[i].size();
- tvf.writeVInt(numTerms);
-
- final TermPositionVector tpVector;
-
- final byte bits;
- final boolean storePositions;
- final boolean storeOffsets;
-
- if (vectors[i] instanceof TermPositionVector) {
- // May have positions & offsets
- tpVector = (TermPositionVector) vectors[i];
- storePositions = tpVector.size() > 0 && tpVector.getTermPositions(0) != null;
- storeOffsets = tpVector.size() > 0 && tpVector.getOffsets(0) != null;
- bits = (byte) ((storePositions ? DefaultTermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR : 0) +
- (storeOffsets ? DefaultTermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR : 0));
- } else {
- tpVector = null;
- bits = 0;
- storePositions = false;
- storeOffsets = false;
- }
-
- tvf.writeVInt(bits);
-
- final BytesRef[] terms = vectors[i].getTerms();
- final int[] freqs = vectors[i].getTermFrequencies();
-
- for (int j=0; j<numTerms; j++) {
-
- int start = j == 0 ? 0 : StringHelper.bytesDifference(terms[j-1].bytes,
- terms[j-1].length,
- terms[j].bytes,
- terms[j].length);
- int length = terms[j].length - start;
- tvf.writeVInt(start); // write shared prefix length
- tvf.writeVInt(length); // write delta length
- tvf.writeBytes(terms[j].bytes, start, length); // write delta bytes
-
- final int termFreq = freqs[j];
-
- tvf.writeVInt(termFreq);
-
- if (storePositions) {
- final int[] positions = tpVector.getTermPositions(j);
- if (positions == null)
- throw new IllegalStateException("Trying to write positions that are null!");
- assert positions.length == termFreq;
-
- // use delta encoding for positions
- int lastPosition = 0;
- for(int k=0;k<positions.length;k++) {
- final int position = positions[k];
- tvf.writeVInt(position-lastPosition);
- lastPosition = position;
- }
- }
-
- if (storeOffsets) {
- final TermVectorOffsetInfo[] offsets = tpVector.getOffsets(j);
- if (offsets == null)
- throw new IllegalStateException("Trying to write offsets that are null!");
- assert offsets.length == termFreq;
-
- // use delta encoding for offsets
- int lastEndOffset = 0;
- for(int k=0;k<offsets.length;k++) {
- final int startOffset = offsets[k].getStartOffset();
- final int endOffset = offsets[k].getEndOffset();
- tvf.writeVInt(startOffset-lastEndOffset);
- tvf.writeVInt(endOffset-startOffset);
- lastEndOffset = endOffset;
- }
- }
- }
- }
-
- // 2nd pass: write field pointers to tvd
- if (numFields > 1) {
- long lastFieldPointer = fieldPointers[0];
- for (int i=1; i<numFields; i++) {
- final long fieldPointer = fieldPointers[i];
- tvd.writeVLong(fieldPointer-lastFieldPointer);
- lastFieldPointer = fieldPointer;
- }
- }
- } else
- tvd.writeVInt(0);
- }
-
+
@Override
public void startDocument(int numVectorFields) throws IOException {
this.numVectorFields = numVectorFields;
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/TermVectorsWriter.java?rev=1200724&r1=1200723&r2=1200724&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/TermVectorsWriter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/TermVectorsWriter.java Fri Nov 11 03:38:27 2011
@@ -24,6 +24,8 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.TermFreqVector;
+import org.apache.lucene.index.TermPositionVector;
+import org.apache.lucene.index.TermVectorOffsetInfo;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -97,7 +99,6 @@ public abstract class TermVectorsWriter
* returning the number of documents that were written.
* Implementations can override this method for more sophisticated
* merging (bulk-byte copying, etc). */
- // nocommit: test that I work
public int merge(MergeState mergeState) throws IOException {
int docCount = 0;
for (MergeState.IndexReaderAndLiveDocs reader : mergeState.readers) {
@@ -120,6 +121,61 @@ public abstract class TermVectorsWriter
return docCount;
}
- // nocommit: this should be a sugar method only that consumes the normal api (once we have one)
- public abstract void addAllDocVectors(TermFreqVector[] vectors, FieldInfos fieldInfos) throws IOException;
+ /** sugar method to write every vector field in the document */
+ protected final void addAllDocVectors(TermFreqVector[] vectors, FieldInfos fieldInfos) throws IOException {
+ final int numFields = vectors == null ? 0 : vectors.length;
+ startDocument(numFields);
+
+ for (int i = 0; i < numFields; i++) {
+ final TermFreqVector vector = vectors[i];
+
+ FieldInfo fieldInfo = fieldInfos.fieldInfo(vector.getField());
+
+ final int numTerms = vectors[i].size();
+ final TermPositionVector posVector;
+ final boolean positions;
+ final boolean offsets;
+
+ if (vector instanceof TermPositionVector) {
+ // May have positions & offsets
+ posVector = (TermPositionVector) vector;
+ positions = posVector.size() > 0 && posVector.getTermPositions(0) != null;
+ offsets = posVector.size() > 0 && posVector.getOffsets(0) != null;
+ } else {
+ posVector = null;
+ positions = offsets = false;
+ }
+
+ startField(fieldInfo, numTerms, positions, offsets);
+
+ final BytesRef[] terms = vectors[i].getTerms();
+ final int[] freqs = vectors[i].getTermFrequencies();
+
+ for (int j = 0; j < numTerms; j++) {
+ startTerm(terms[j], freqs[j]);
+
+ if (positions) {
+ final int[] pos = posVector.getTermPositions(j);
+ if (pos == null) {
+ throw new IllegalStateException("Trying to write positions that are null!");
+ }
+ assert pos.length == freqs[j];
+ for (int k = 0; k < pos.length; k++) {
+ addPosition(pos[k]);
+ }
+ }
+
+ if (offsets) {
+ final TermVectorOffsetInfo[] off = posVector.getOffsets(j);
+ if (off == null) {
+ throw new IllegalStateException("Trying to write offsets that are null!");
+ }
+ assert off.length == freqs[j];
+ for (int k = 0; k < off.length; k++) {
+ addOffset(off[k].getStartOffset(), off[k].getEndOffset());
+ }
+ }
+ }
+ }
+ }
}