You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2013/01/21 20:11:59 UTC
svn commit: r1436556 [1/2] - in /lucene/dev/trunk/lucene: ./
core/src/java/org/apache/lucene/codecs/compressing/
core/src/java/org/apache/lucene/util/packed/
core/src/test/org/apache/lucene/index/
test-framework/src/java/org/apache/lucene/codecs/compre...
Author: jpountz
Date: Mon Jan 21 19:11:59 2013
New Revision: 1436556
URL: http://svn.apache.org/viewvc?rev=1436556&view=rev
Log:
LUCENE-4599: New compressed TVF impl: CompressingTermVectorsFormat.
Added:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java (with props)
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java (with props)
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java (with props)
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1436556&r1=1436555&r2=1436556&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Mon Jan 21 19:11:59 2013
@@ -61,6 +61,10 @@ New Features
* LUCENE-4703: Add simple PrintTaxonomyStats tool to see summary
information about the facets taxonomy index. (Mike McCandless)
+* LUCENE-4599: New oal.codecs.compressing.CompressingTermVectorsFormat which
+ compresses term vectors into chunks of documents similarly to
+ CompressingStoredFieldsFormat. (Adrien Grand)
+
======================= Lucene 4.1.0 =======================
Changes in backwards compatibility policy
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java?rev=1436556&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java Mon Jan 21 19:11:59 2013
@@ -0,0 +1,102 @@
+package org.apache.lucene.codecs.compressing;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.TermVectorsReader;
+import org.apache.lucene.codecs.TermVectorsWriter;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+
+/**
+ * A {@link TermVectorsFormat} that compresses chunks of documents together in
+ * order to improve the compression ratio.
+ * @lucene.experimental
+ */
+public final class CompressingTermVectorsFormat extends TermVectorsFormat {
+
+ private final String formatName;
+ private final String segmentSuffix;
+ private final CompressionMode compressionMode;
+ private final int chunkSize;
+
+ /**
+ * Create a new {@link CompressingTermVectorsFormat}.
+ * <p>
+ * <code>formatName</code> is the name of the format. This name will be used
+ * in the file formats to perform
+ * {@link CodecUtil#checkHeader(org.apache.lucene.store.DataInput, String, int, int) codec header checks}.
+ * <p>
+ * The <code>compressionMode</code> parameter allows you to choose between
+ * compression algorithms that have various compression and decompression
+ * speeds so that you can pick the one that best fits your indexing and
+ * searching throughput. You should never instantiate two
+ * {@link CompressingTermVectorsFormat}s that have the same name but
+ * different {@link CompressionMode}s.
+ * <p>
+ * <code>chunkSize</code> is the minimum byte size of a chunk of documents.
+ * Higher values of <code>chunkSize</code> should improve the compression
+ * ratio but will require more memory at indexing time and might make document
+ * loading a little slower (depending on the size of your OS cache compared
+ * to the size of your index).
+ *
+ * @param formatName the name of the {@link StoredFieldsFormat}
+ * @param segmentSuffix a suffix to append to files created by this format
+ * @param compressionMode the {@link CompressionMode} to use
+ * @param chunkSize the minimum number of bytes of a single chunk of stored documents
+ * @see CompressionMode
+ */
+ public CompressingTermVectorsFormat(String formatName, String segmentSuffix,
+ CompressionMode compressionMode, int chunkSize) {
+ this.formatName = formatName;
+ this.segmentSuffix = segmentSuffix;
+ this.compressionMode = compressionMode;
+ if (chunkSize < 1) {
+ throw new IllegalArgumentException("chunkSize must be >= 1");
+ }
+ this.chunkSize = chunkSize;
+ }
+
+ @Override
+ public TermVectorsReader vectorsReader(Directory directory,
+ SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
+ throws IOException {
+ return new CompressingTermVectorsReader(directory, segmentInfo, segmentSuffix,
+ fieldInfos, context, formatName, compressionMode);
+ }
+
+ @Override
+ public TermVectorsWriter vectorsWriter(Directory directory,
+ SegmentInfo segmentInfo, IOContext context) throws IOException {
+ return new CompressingTermVectorsWriter(directory, segmentInfo, segmentSuffix,
+ context, formatName, compressionMode, chunkSize);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(compressionMode=" + compressionMode
+ + ", chunkSize=" + chunkSize + ")";
+ }
+
+}
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java?rev=1436556&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java Mon Jan 21 19:11:59 2013
@@ -0,0 +1,1048 @@
+package org.apache.lucene.codecs.compressing;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.BLOCK_SIZE;
+import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.CODEC_SFX_DAT;
+import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.CODEC_SFX_IDX;
+import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.FLAGS_BITS;
+import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.OFFSETS;
+import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.PAYLOADS;
+import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.POSITIONS;
+import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VECTORS_EXTENSION;
+import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VECTORS_INDEX_EXTENSION;
+import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_CURRENT;
+import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_START;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.TermVectorsReader;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LongsRef;
+import org.apache.lucene.util.packed.BlockPackedReader;
+import org.apache.lucene.util.packed.PackedInts;
+
+
+/**
+ * {@link TermVectorsReader} for {@link CompressingTermVectorsFormat}.
+ * @lucene.experimental
+ */
+public final class CompressingTermVectorsReader extends TermVectorsReader implements Closeable {
+
+ private final FieldInfos fieldInfos;
+ final CompressingStoredFieldsIndexReader indexReader;
+ final IndexInput vectorsStream;
+ private final int packedIntsVersion;
+ private final CompressionMode compressionMode;
+ private final Decompressor decompressor;
+ private final int chunkSize;
+ private final int numDocs;
+ private boolean closed;
+ private final BlockPackedReader reader;
+
+ // used by clone
+ private CompressingTermVectorsReader(CompressingTermVectorsReader reader) {
+ this.fieldInfos = reader.fieldInfos;
+ this.vectorsStream = reader.vectorsStream.clone();
+ this.indexReader = reader.indexReader.clone();
+ this.packedIntsVersion = reader.packedIntsVersion;
+ this.compressionMode = reader.compressionMode;
+ this.decompressor = reader.decompressor.clone();
+ this.chunkSize = reader.chunkSize;
+ this.numDocs = reader.numDocs;
+ this.reader = new BlockPackedReader(vectorsStream, packedIntsVersion, BLOCK_SIZE, 0);
+ this.closed = false;
+ }
+
+ /** Sole constructor. */
+ public CompressingTermVectorsReader(Directory d, SegmentInfo si, String segmentSuffix, FieldInfos fn,
+ IOContext context, String formatName, CompressionMode compressionMode) throws IOException {
+ this.compressionMode = compressionMode;
+ final String segment = si.name;
+ boolean success = false;
+ fieldInfos = fn;
+ numDocs = si.getDocCount();
+ IndexInput indexStream = null;
+ try {
+ vectorsStream = d.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION), context);
+ final String indexStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_INDEX_EXTENSION);
+ indexStream = d.openInput(indexStreamFN, context);
+
+ final String codecNameIdx = formatName + CODEC_SFX_IDX;
+ final String codecNameDat = formatName + CODEC_SFX_DAT;
+ CodecUtil.checkHeader(indexStream, codecNameIdx, VERSION_START, VERSION_CURRENT);
+ CodecUtil.checkHeader(vectorsStream, codecNameDat, VERSION_START, VERSION_CURRENT);
+ assert CodecUtil.headerLength(codecNameDat) == vectorsStream.getFilePointer();
+ assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer();
+
+ indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
+ indexStream = null;
+
+ packedIntsVersion = vectorsStream.readVInt();
+ chunkSize = vectorsStream.readVInt();
+ decompressor = compressionMode.newDecompressor();
+ this.reader = new BlockPackedReader(vectorsStream, packedIntsVersion, BLOCK_SIZE, 0);
+
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(this, indexStream);
+ }
+ }
+ }
+
+ CompressionMode getCompressionMode() {
+ return compressionMode;
+ }
+
+ int getChunkSize() {
+ return chunkSize;
+ }
+
+ int getPackedIntsVersion() {
+ return packedIntsVersion;
+ }
+
+ CompressingStoredFieldsIndexReader getIndex() {
+ return indexReader;
+ }
+
+ IndexInput getVectorsStream() {
+ return vectorsStream;
+ }
+
+ /**
+ * @throws AlreadyClosedException if this TermVectorsReader is closed
+ */
+ private void ensureOpen() throws AlreadyClosedException {
+ if (closed) {
+ throw new AlreadyClosedException("this FieldsReader is closed");
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (!closed) {
+ IOUtils.close(vectorsStream, indexReader);
+ closed = true;
+ }
+ }
+
+ @Override
+ public TermVectorsReader clone() {
+ return new CompressingTermVectorsReader(this);
+ }
+
+ @Override
+ public Fields get(int doc) throws IOException {
+ ensureOpen();
+
+ // seek to the right place
+ {
+ final long startPointer = indexReader.getStartPointer(doc);
+ vectorsStream.seek(startPointer);
+ }
+
+ // decode
+ // - docBase: first doc ID of the chunk
+ // - chunkDocs: number of docs of the chunk
+ final int docBase = vectorsStream.readVInt();
+ final int chunkDocs = vectorsStream.readVInt();
+ if (doc < docBase || doc >= docBase + chunkDocs || docBase + chunkDocs > numDocs) {
+ throw new CorruptIndexException("docBase=" + docBase + ",chunkDocs=" + chunkDocs + ",doc=" + doc);
+ }
+
+ final int skip; // number of fields to skip
+ final int numFields; // number of fields of the document we're looking for
+ final int totalFields; // total number of fields of the chunk (sum for all docs)
+ if (chunkDocs == 1) {
+ skip = 0;
+ numFields = totalFields = vectorsStream.readVInt();
+ } else {
+ reader.reset(vectorsStream, chunkDocs);
+ int sum = 0;
+ for (int i = docBase; i < doc; ++i) {
+ sum += reader.next();
+ }
+ skip = sum;
+ numFields = (int) reader.next();
+ sum += numFields;
+ for (int i = doc + 1; i < docBase + chunkDocs; ++i) {
+ sum += reader.next();
+ }
+ totalFields = sum;
+ }
+
+ if (numFields == 0) {
+ // no vectors
+ return null;
+ }
+
+ // read field numbers that have term vectors
+ final int[] fieldNums;
+ {
+ final int token = vectorsStream.readByte() & 0xFF;
+ assert token != 0; // means no term vectors, cannot happen since we checked for numFields == 0
+ final int bitsPerFieldNum = token & 0x1F;
+ int totalDistinctFields = token >>> 5;
+ if (totalDistinctFields == 0x07) {
+ totalDistinctFields += vectorsStream.readVInt();
+ }
+ ++totalDistinctFields;
+ final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalDistinctFields, bitsPerFieldNum, 1);
+ fieldNums = new int[totalDistinctFields];
+ for (int i = 0; i < totalDistinctFields; ++i) {
+ fieldNums[i] = (int) it.next();
+ }
+ }
+
+ // read field numbers and flags
+ final int[] fieldNumOffs = new int[numFields];
+ final PackedInts.Reader flags;
+ {
+ final int bitsPerOff = PackedInts.bitsRequired(fieldNums.length - 1);
+ final PackedInts.Reader allFieldNumOffs = PackedInts.getReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, bitsPerOff);
+ switch (vectorsStream.readVInt()) {
+ case 0:
+ final PackedInts.Reader fieldFlags = PackedInts.getReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, fieldNums.length, FLAGS_BITS);
+ PackedInts.Mutable f = PackedInts.getMutable(totalFields, FLAGS_BITS, PackedInts.COMPACT);
+ for (int i = 0; i < totalFields; ++i) {
+ final int fieldNumOff = (int) allFieldNumOffs.get(i);
+ assert fieldNumOff >= 0 && fieldNumOff < fieldNums.length;
+ final int fgs = (int) fieldFlags.get(fieldNumOff);
+ f.set(i, fgs);
+ }
+ flags = f;
+ break;
+ case 1:
+ flags = PackedInts.getReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, FLAGS_BITS);
+ break;
+ default:
+ throw new AssertionError();
+ }
+ for (int i = 0; i < numFields; ++i) {
+ fieldNumOffs[i] = (int) allFieldNumOffs.get(skip + i);
+ }
+ }
+
+ // number of terms per field for all fields
+ final PackedInts.Reader numTerms;
+ final int totalTerms;
+ {
+ final int bitsRequired = vectorsStream.readVInt();
+ numTerms = PackedInts.getReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, bitsRequired);
+ int sum = 0;
+ for (int i = 0; i < totalFields; ++i) {
+ sum += numTerms.get(i);
+ }
+ totalTerms = sum;
+ }
+
+ // term lengths
+ int docOff = 0, docLen = 0, totalLen;
+ final int[] fieldLengths = new int[numFields];
+ final int[][] prefixLengths = new int[numFields][];
+ final int[][] suffixLengths = new int[numFields][];
+ {
+ reader.reset(vectorsStream, totalTerms);
+ // skip
+ int toSkip = 0;
+ for (int i = 0; i < skip; ++i) {
+ toSkip += numTerms.get(i);
+ }
+ reader.skip(toSkip);
+ // read prefix lengths
+ for (int i = 0; i < numFields; ++i) {
+ final int termCount = (int) numTerms.get(skip + i);
+ final int[] fieldPrefixLengths = new int[termCount];
+ prefixLengths[i] = fieldPrefixLengths;
+ for (int j = 0; j < termCount; ) {
+ final LongsRef next = reader.next(termCount - j);
+ for (int k = 0; k < next.length; ++k) {
+ fieldPrefixLengths[j++] = (int) next.longs[next.offset + k];
+ }
+ }
+ }
+ reader.skip(totalTerms - reader.ord());
+
+ reader.reset(vectorsStream, totalTerms);
+ // skip
+ toSkip = 0;
+ for (int i = 0; i < skip; ++i) {
+ for (int j = 0; j < numTerms.get(i); ++j) {
+ docOff += reader.next();
+ }
+ }
+ for (int i = 0; i < numFields; ++i) {
+ final int termCount = (int) numTerms.get(skip + i);
+ final int[] fieldSuffixLengths = new int[termCount];
+ suffixLengths[i] = fieldSuffixLengths;
+ for (int j = 0; j < termCount; ) {
+ final LongsRef next = reader.next(termCount - j);
+ for (int k = 0; k < next.length; ++k) {
+ fieldSuffixLengths[j++] = (int) next.longs[next.offset + k];
+ }
+ }
+ fieldLengths[i] = sum(suffixLengths[i]);
+ docLen += fieldLengths[i];
+ }
+ totalLen = docOff + docLen;
+ for (int i = skip + numFields; i < totalFields; ++i) {
+ for (int j = 0; j < numTerms.get(i); ++j) {
+ totalLen += reader.next();
+ }
+ }
+ }
+
+ // term freqs
+ final int[] termFreqs = new int[totalTerms];
+ {
+ reader.reset(vectorsStream, totalTerms);
+ for (int i = 0; i < totalTerms; ) {
+ final LongsRef next = reader.next(totalTerms - i);
+ for (int k = 0; k < next.length; ++k) {
+ termFreqs[i++] = 1 + (int) next.longs[next.offset + k];
+ }
+ }
+ }
+
+ // total number of positions, offsets and payloads
+ int totalPositions = 0, totalOffsets = 0, totalPayloads = 0;
+ for (int i = 0, termIndex = 0; i < totalFields; ++i) {
+ final int f = (int) flags.get(i);
+ final int termCount = (int) numTerms.get(i);
+ for (int j = 0; j < termCount; ++j) {
+ final int freq = termFreqs[termIndex++];
+ if ((f & POSITIONS) != 0) {
+ totalPositions += freq;
+ }
+ if ((f & OFFSETS) != 0) {
+ totalOffsets += freq;
+ }
+ if ((f & PAYLOADS) != 0) {
+ totalPayloads += freq;
+ }
+ }
+ assert i != totalFields - 1 || termIndex == totalTerms : termIndex + " " + totalTerms;
+ }
+
+ final int[][] positionIndex = positionIndex(skip, numFields, numTerms, termFreqs);
+ final int[][] positions, startOffsets, lengths;
+ if (totalPositions > 0) {
+ positions = readPositions(skip, numFields, flags, numTerms, termFreqs, POSITIONS, totalPositions, positionIndex);
+ } else {
+ positions = new int[numFields][];
+ }
+
+ if (totalOffsets > 0) {
+ // average number of chars per term
+ final float[] charsPerTerm = new float[fieldNums.length];
+ for (int i = 0; i < charsPerTerm.length; ++i) {
+ charsPerTerm[i] = Float.intBitsToFloat(vectorsStream.readInt());
+ }
+ startOffsets = readPositions(skip, numFields, flags, numTerms, termFreqs, OFFSETS, totalOffsets, positionIndex);
+ lengths = readPositions(skip, numFields, flags, numTerms, termFreqs, OFFSETS, totalOffsets, positionIndex);
+
+ for (int i = 0; i < numFields; ++i) {
+ final int[] fStartOffsets = startOffsets[i];
+ final int[] fPositions = positions[i];
+ // patch offsets from positions
+ if (fStartOffsets != null && fPositions != null) {
+ final float fieldCharsPerTerm = charsPerTerm[fieldNumOffs[i]];
+ for (int j = 0; j < startOffsets[i].length; ++j) {
+ fStartOffsets[j] += (int) (fieldCharsPerTerm * fPositions[j]);
+ }
+ }
+ if (fStartOffsets != null) {
+ final int[] fPrefixLengths = prefixLengths[i];
+ final int[] fSuffixLengths = suffixLengths[i];
+ final int[] fLengths = lengths[i];
+ for (int j = 0, end = (int) numTerms.get(skip + i); j < end; ++j) {
+ // delta-decode start offsets and patch lengths using term lengths
+ final int termLength = fPrefixLengths[j] + fSuffixLengths[j];
+ lengths[i][positionIndex[i][j]] += termLength;
+ for (int k = positionIndex[i][j] + 1; k < positionIndex[i][j + 1]; ++k) {
+ fStartOffsets[k] += fStartOffsets[k - 1];
+ fLengths[k] += termLength;
+ }
+ }
+ }
+ }
+ } else {
+ startOffsets = lengths = new int[numFields][];
+ }
+ if (totalPositions > 0) {
+ // delta-decode positions
+ for (int i = 0; i < numFields; ++i) {
+ final int[] fPositions = positions[i];
+ final int[] fpositionIndex = positionIndex[i];
+ if (fPositions != null) {
+ for (int j = 0, end = (int) numTerms.get(skip + i); j < end; ++j) {
+ // delta-decode start offsets
+ for (int k = fpositionIndex[j] + 1; k < fpositionIndex[j + 1]; ++k) {
+ fPositions[k] += fPositions[k - 1];
+ }
+ }
+ }
+ }
+ }
+
+ // payload lengths
+ final int[][] payloadIndex = new int[numFields][];
+ int totalPayloadLength = 0;
+ int payloadOff = 0;
+ int payloadLen = 0;
+ if (totalPayloads > 0) {
+ reader.reset(vectorsStream, totalPayloads);
+ // skip
+ int termIndex = 0;
+ for (int i = 0; i < skip; ++i) {
+ final int f = (int) flags.get(i);
+ final int termCount = (int) numTerms.get(i);
+ if ((f & PAYLOADS) != 0) {
+ for (int j = 0; j < termCount; ++j) {
+ final int freq = termFreqs[termIndex + j];
+ for (int k = 0; k < freq; ++k) {
+ final int l = (int) reader.next();
+ payloadOff += l;
+ }
+ }
+ }
+ termIndex += termCount;
+ }
+ totalPayloadLength = payloadOff;
+ // read doc payload lengths
+ for (int i = 0; i < numFields; ++i) {
+ final int f = (int) flags.get(skip + i);
+ final int termCount = (int) numTerms.get(skip + i);
+ if ((f & PAYLOADS) != 0) {
+ final int totalFreq = positionIndex[i][termCount];
+ payloadIndex[i] = new int[totalFreq + 1];
+ int posIdx = 0;
+ payloadIndex[i][posIdx] = payloadLen;
+ for (int j = 0; j < termCount; ++j) {
+ final int freq = termFreqs[termIndex + j];
+ for (int k = 0; k < freq; ++k) {
+ final int payloadLength = (int) reader.next();
+ payloadLen += payloadLength;
+ payloadIndex[i][posIdx+1] = payloadLen;
+ ++posIdx;
+ }
+ }
+ assert posIdx == totalFreq;
+ }
+ termIndex += termCount;
+ }
+ totalPayloadLength += payloadLen;
+ for (int i = skip + numFields; i < totalFields; ++i) {
+ final int f = (int) flags.get(i);
+ final int termCount = (int) numTerms.get(i);
+ if ((f & PAYLOADS) != 0) {
+ for (int j = 0; j < termCount; ++j) {
+ final int freq = termFreqs[termIndex + j];
+ for (int k = 0; k < freq; ++k) {
+ totalPayloadLength += reader.next();
+ }
+ }
+ }
+ termIndex += termCount;
+ }
+ assert termIndex == totalTerms : termIndex + " " + totalTerms;
+ }
+
+ // decompress data
+ final BytesRef suffixBytes = new BytesRef();
+ decompressor.decompress(vectorsStream, totalLen + totalPayloadLength, docOff + payloadOff, docLen + payloadLen, suffixBytes);
+ suffixBytes.length = docLen;
+ final BytesRef payloadBytes = new BytesRef(suffixBytes.bytes, suffixBytes.offset + docLen, payloadLen);
+
+ final int[] fieldFlags = new int[numFields];
+ for (int i = 0; i < numFields; ++i) {
+ fieldFlags[i] = (int) flags.get(skip + i);
+ }
+
+ final int[] fieldNumTerms = new int[numFields];
+ for (int i = 0; i < numFields; ++i) {
+ fieldNumTerms[i] = (int) numTerms.get(skip + i);
+ }
+
+ final int[][] fieldTermFreqs = new int[numFields][];
+ {
+ int termIdx = 0;
+ for (int i = 0; i < skip; ++i) {
+ termIdx += numTerms.get(i);
+ }
+ for (int i = 0; i < numFields; ++i) {
+ final int termCount = (int) numTerms.get(skip + i);
+ fieldTermFreqs[i] = new int[termCount];
+ for (int j = 0; j < termCount; ++j) {
+ fieldTermFreqs[i][j] = termFreqs[termIdx++];
+ }
+ }
+ }
+
+ assert sum(fieldLengths) == docLen : sum(fieldLengths) + " != " + docLen;
+
+ return new TVFields(fieldNums, fieldFlags, fieldNumOffs, fieldNumTerms, fieldLengths,
+ prefixLengths, suffixLengths, fieldTermFreqs,
+ positionIndex, positions, startOffsets, lengths,
+ payloadBytes, payloadIndex,
+ suffixBytes);
+ }
+
+ // field -> term index -> position index
+ private int[][] positionIndex(int skip, int numFields, PackedInts.Reader numTerms, int[] termFreqs) {
+ final int[][] positionIndex = new int[numFields][];
+ int termIndex = 0;
+ for (int i = 0; i < skip; ++i) {
+ final int termCount = (int) numTerms.get(i);
+ termIndex += termCount;
+ }
+ for (int i = 0; i < numFields; ++i) {
+ final int termCount = (int) numTerms.get(skip + i);
+ positionIndex[i] = new int[termCount + 1];
+ for (int j = 0; j < termCount; ++j) {
+ final int freq = termFreqs[termIndex+j];
+ positionIndex[i][j + 1] = positionIndex[i][j] + freq;
+ }
+ termIndex += termCount;
+ }
+ return positionIndex;
+ }
+
+ private int[][] readPositions(int skip, int numFields, PackedInts.Reader flags, PackedInts.Reader numTerms, int[] termFreqs, int flag, final int totalPositions, int[][] positionIndex) throws IOException {
+ final int[][] positions = new int[numFields][];
+ reader.reset(vectorsStream, totalPositions);
+ // skip
+ int toSkip = 0;
+ int termIndex = 0;
+ for (int i = 0; i < skip; ++i) {
+ final int f = (int) flags.get(i);
+ final int termCount = (int) numTerms.get(i);
+ if ((f & flag) != 0) {
+ for (int j = 0; j < termCount; ++j) {
+ final int freq = termFreqs[termIndex+j];
+ toSkip += freq;
+ }
+ }
+ termIndex += termCount;
+ }
+ reader.skip(toSkip);
+ // read doc positions
+ for (int i = 0; i < numFields; ++i) {
+ final int f = (int) flags.get(skip + i);
+ final int termCount = (int) numTerms.get(skip + i);
+ if ((f & flag) != 0) {
+ final int totalFreq = positionIndex[i][termCount];
+ final int[] fieldPositions = new int[totalFreq];
+ positions[i] = fieldPositions;
+ for (int j = 0; j < totalFreq; ) {
+ final LongsRef nextPositions = reader.next(totalFreq - j);
+ for (int k = 0; k < nextPositions.length; ++k) {
+ fieldPositions[j++] = (int) nextPositions.longs[nextPositions.offset + k];
+ }
+ }
+ }
+ termIndex += termCount;
+ }
+ reader.skip(totalPositions - reader.ord());
+ return positions;
+ }
+
+ private class TVFields extends Fields {
+
+ private final int[] fieldNums, fieldFlags, fieldNumOffs, numTerms, fieldLengths;
+ private final int[][] prefixLengths, suffixLengths, termFreqs, positionIndex, positions, startOffsets, lengths, payloadIndex;
+ private final BytesRef suffixBytes, payloadBytes;
+
+ public TVFields(int[] fieldNums, int[] fieldFlags, int[] fieldNumOffs, int[] numTerms, int[] fieldLengths,
+ int[][] prefixLengths, int[][] suffixLengths, int[][] termFreqs,
+ int[][] positionIndex, int[][] positions, int[][] startOffsets, int[][] lengths,
+ BytesRef payloadBytes, int[][] payloadIndex,
+ BytesRef suffixBytes) {
+ this.fieldNums = fieldNums;
+ this.fieldFlags = fieldFlags;
+ this.fieldNumOffs = fieldNumOffs;
+ this.numTerms = numTerms;
+ this.fieldLengths = fieldLengths;
+ this.prefixLengths = prefixLengths;
+ this.suffixLengths = suffixLengths;
+ this.termFreqs = termFreqs;
+ this.positionIndex = positionIndex;
+ this.positions = positions;
+ this.startOffsets = startOffsets;
+ this.lengths = lengths;
+ this.payloadBytes = payloadBytes;
+ this.payloadIndex = payloadIndex;
+ this.suffixBytes = suffixBytes;
+ }
+
+ @Override
+ public Iterator<String> iterator() {
+ return new Iterator<String>() {
+ int i = 0;
+ @Override
+ public boolean hasNext() {
+ return i < fieldNumOffs.length;
+ }
+ @Override
+ public String next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ final int fieldNum = fieldNums[fieldNumOffs[i++]];
+ return fieldInfos.fieldInfo(fieldNum).name;
+ }
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ @Override
+ public Terms terms(String field) throws IOException {
+ final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ if (fieldInfo == null) {
+ return null;
+ }
+ int idx = -1;
+ for (int i = 0; i < fieldNumOffs.length; ++i) {
+ if (fieldNums[fieldNumOffs[i]] == fieldInfo.number) {
+ idx = i;
+ break;
+ }
+ }
+
+ if (idx == -1 || numTerms[idx] == 0) {
+ // no term
+ return null;
+ }
+ int fieldOff = 0, fieldLen = -1;
+ for (int i = 0; i < fieldNumOffs.length; ++i) {
+ if (i < idx) {
+ fieldOff += fieldLengths[i];
+ } else {
+ fieldLen = fieldLengths[i];
+ break;
+ }
+ }
+ assert fieldLen >= 0;
+ return new TVTerms(numTerms[idx], fieldFlags[idx],
+ prefixLengths[idx], suffixLengths[idx], termFreqs[idx],
+ positionIndex[idx], positions[idx], startOffsets[idx], lengths[idx],
+ payloadIndex[idx], payloadBytes,
+ new BytesRef(suffixBytes.bytes, suffixBytes.offset + fieldOff, fieldLen));
+ }
+
+ @Override
+ public int size() {
+ return fieldNumOffs.length;
+ }
+
+ }
+
+ private class TVTerms extends Terms {
+
+ private final int numTerms, flags;
+ private final int[] prefixLengths, suffixLengths, termFreqs, positionIndex, positions, startOffsets, lengths, payloadIndex;
+ private final BytesRef termBytes, payloadBytes;
+
+ TVTerms(int numTerms, int flags, int[] prefixLengths, int[] suffixLengths, int[] termFreqs,
+ int[] positionIndex, int[] positions, int[] startOffsets, int[] lengths,
+ int[] payloadIndex, BytesRef payloadBytes,
+ BytesRef termBytes) {
+ this.numTerms = numTerms;
+ this.flags = flags;
+ this.prefixLengths = prefixLengths;
+ this.suffixLengths = suffixLengths;
+ this.termFreqs = termFreqs;
+ this.positionIndex = positionIndex;
+ this.positions = positions;
+ this.startOffsets = startOffsets;
+ this.lengths = lengths;
+ this.payloadIndex = payloadIndex;
+ this.payloadBytes = payloadBytes;
+ this.termBytes = termBytes;
+ }
+
+ @Override
+ public TermsEnum iterator(TermsEnum reuse) throws IOException {
+ final TVTermsEnum termsEnum;
+ if (reuse != null && reuse instanceof TVTermsEnum) {
+ termsEnum = (TVTermsEnum) reuse;
+ } else {
+ termsEnum = new TVTermsEnum();
+ }
+ termsEnum.reset(numTerms, flags, prefixLengths, suffixLengths, termFreqs, positionIndex, positions, startOffsets, lengths,
+ payloadIndex, payloadBytes,
+ new ByteArrayDataInput(termBytes.bytes, termBytes.offset, termBytes.length));
+ return termsEnum;
+ }
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ }
+
+ @Override
+ public long size() throws IOException {
+ return numTerms;
+ }
+
+ @Override
+ public long getSumTotalTermFreq() throws IOException {
+ return -1L;
+ }
+
+ @Override
+ public long getSumDocFreq() throws IOException {
+ return numTerms;
+ }
+
+ @Override
+ public int getDocCount() throws IOException {
+ return 1;
+ }
+
+ @Override
+ public boolean hasOffsets() {
+ return (flags & OFFSETS) != 0;
+ }
+
+ @Override
+ public boolean hasPositions() {
+ return (flags & POSITIONS) != 0;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return (flags & PAYLOADS) != 0;
+ }
+
+ }
+
+ private static class TVTermsEnum extends TermsEnum {
+
+ private int numTerms, startPos, ord;
+ private int[] prefixLengths, suffixLengths, termFreqs, positionIndex, positions, startOffsets, lengths, payloadIndex;
+ private ByteArrayDataInput in;
+ private BytesRef payloads;
+ private final BytesRef term;
+
+ private TVTermsEnum() {
+ term = new BytesRef(16);
+ }
+
+ void reset(int numTerms, int flags, int[] prefixLengths, int[] suffixLengths, int[] termFreqs, int[] positionIndex, int[] positions, int[] startOffsets, int[] lengths,
+ int[] payloadIndex, BytesRef payloads, ByteArrayDataInput in) {
+ this.numTerms = numTerms;
+ this.prefixLengths = prefixLengths;
+ this.suffixLengths = suffixLengths;
+ this.termFreqs = termFreqs;
+ this.positionIndex = positionIndex;
+ this.positions = positions;
+ this.startOffsets = startOffsets;
+ this.lengths = lengths;
+ this.payloadIndex = payloadIndex;
+ this.payloads = payloads;
+ this.in = in;
+ startPos = in.getPosition();
+ reset();
+ }
+
+ void reset() {
+ term.length = 0;
+ in.setPosition(startPos);
+ ord = -1;
+ }
+
+ @Override
+ public BytesRef next() throws IOException {
+ if (ord == numTerms - 1) {
+ return null;
+ } else {
+ assert ord < numTerms;
+ ++ord;
+ }
+
+ // read term
+ term.offset = 0;
+ term.length = prefixLengths[ord] + suffixLengths[ord];
+ if (term.length > term.bytes.length) {
+ term.bytes = ArrayUtil.grow(term.bytes, term.length);
+ }
+ in.readBytes(term.bytes, prefixLengths[ord], suffixLengths[ord]);
+
+ return term;
+ }
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ }
+
+ @Override
+ public SeekStatus seekCeil(BytesRef text, boolean useCache)
+ throws IOException {
+ if (ord < numTerms && ord >= 0) {
+ final int cmp = term().compareTo(text);
+ if (cmp == 0) {
+ return SeekStatus.FOUND;
+ } else if (cmp > 0) {
+ reset();
+ }
+ }
+ // linear scan
+ do {
+ next();
+ } while (ord < numTerms - 1 && term().compareTo(text) < 0);
+ return term().equals(text) ? SeekStatus.FOUND : SeekStatus.END;
+ }
+
+ @Override
+ public void seekExact(long ord) throws IOException {
+ if (ord < -1 || ord >= numTerms) {
+ throw new IOException("ord is out of range: ord=" + ord + ", numTerms=" + numTerms);
+ }
+ if (ord < this.ord) {
+ reset();
+ }
+ for (int i = this.ord; i < ord; ++i) {
+ next();
+ }
+ assert ord == this.ord();
+ }
+
+ @Override
+ public BytesRef term() throws IOException {
+ return term;
+ }
+
+ @Override
+ public long ord() throws IOException {
+ return ord;
+ }
+
+ @Override
+ public int docFreq() throws IOException {
+ return 1;
+ }
+
+ @Override
+ public long totalTermFreq() throws IOException {
+ return termFreqs[ord];
+ }
+
+ @Override
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags)
+ throws IOException {
+ return docsAndPositions(liveDocs, reuse, flags);
+ }
+
+ @Override
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs,
+ DocsAndPositionsEnum reuse, int flags) throws IOException {
+ if ((flags & POSITIONS) == 0 && (flags & OFFSETS) == 0) {
+ return null;
+ }
+ return docsAndPositions(liveDocs, (DocsEnum) reuse, flags);
+ }
+
+ private DocsAndPositionsEnum docsAndPositions(Bits liveDocs,
+ DocsEnum reuse, int ignoredFlags) throws IOException {
+ final TVDocsEnum docsEnum;
+ if (reuse != null && reuse instanceof TVDocsEnum) {
+ docsEnum = (TVDocsEnum) reuse;
+ } else {
+ docsEnum = new TVDocsEnum();
+ }
+
+ docsEnum.reset(liveDocs, termFreqs[ord], positionIndex[ord], positions, startOffsets, lengths, payloads, payloadIndex);
+ return docsEnum;
+ }
+
+ }
+
+ private static class TVDocsEnum extends DocsAndPositionsEnum {
+
+ private Bits liveDocs;
+ private int doc = -1;
+ private int termFreq;
+ private int positionIndex;
+ private int[] positions;
+ private int[] startOffsets;
+ private int[] lengths;
+ private final BytesRef payload;
+ private int[] payloadIndex;
+ private int basePayloadOffset;
+ private int i;
+
+ TVDocsEnum() {
+ payload = new BytesRef();
+ }
+
+ public void reset(Bits liveDocs, int freq, int positionIndex, int[] positions,
+ int[] startOffsets, int[] lengths, BytesRef payloads,
+ int[] payloadIndex) {
+ this.liveDocs = liveDocs;
+ this.termFreq = freq;
+ this.positionIndex = positionIndex;
+ this.positions = positions;
+ this.startOffsets = startOffsets;
+ this.lengths = lengths;
+ this.basePayloadOffset = payloads.offset;
+ this.payload.bytes = payloads.bytes;
+ payload.offset = payload.length = 0;
+ this.payloadIndex = payloadIndex;
+
+ doc = i = -1;
+ }
+
+ private void checkDoc() {
+ if (doc == NO_MORE_DOCS) {
+ throw new IllegalStateException("DocsEnum exhausted");
+ } else if (doc == -1) {
+ throw new IllegalStateException("DocsEnum not started");
+ }
+ }
+
+ private void checkPosition() {
+ checkDoc();
+ if (i < 0) {
+ throw new IllegalStateException("Position enum not started");
+ } else if (i >= termFreq) {
+ throw new IllegalStateException("Read past last position");
+ }
+ }
+
+ @Override
+ public int nextPosition() throws IOException {
+ if (doc != 0) {
+ throw new IllegalStateException();
+ } else if (i >= termFreq - 1) {
+ throw new IllegalStateException("Read past last position");
+ }
+
+ ++i;
+
+ if (payloadIndex != null) {
+ payload.offset = basePayloadOffset + payloadIndex[positionIndex + i];
+ payload.length = payloadIndex[positionIndex + i + 1] - payloadIndex[positionIndex + i];
+ }
+
+ if (positions == null) {
+ return -1;
+ } else {
+ return positions[positionIndex + i];
+ }
+ }
+
+ @Override
+ public int startOffset() throws IOException {
+ checkPosition();
+ if (startOffsets == null) {
+ return -1;
+ } else {
+ return startOffsets[positionIndex + i];
+ }
+ }
+
+ @Override
+ public int endOffset() throws IOException {
+ checkPosition();
+ if (startOffsets == null) {
+ return -1;
+ } else {
+ return startOffsets[positionIndex + i] + lengths[positionIndex + i];
+ }
+ }
+
+ @Override
+ public BytesRef getPayload() throws IOException {
+ checkPosition();
+ if (payloadIndex == null || payload.length == 0) {
+ return null;
+ } else {
+ return payload;
+ }
+ }
+
+ @Override
+ public int freq() throws IOException {
+ checkDoc();
+ return termFreq;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (doc == -1 && (liveDocs == null || liveDocs.get(0))) {
+ return (doc = 0);
+ } else {
+ return (doc = NO_MORE_DOCS);
+ }
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (doc == -1 && target == 0 && (liveDocs == null || liveDocs.get(0))) {
+ return (doc = 0);
+ } else {
+ return (doc = NO_MORE_DOCS);
+ }
+ }
+
+ }
+
+ private static int sum(int[] arr) {
+ int sum = 0;
+ for (int el : arr) {
+ sum += el;
+ }
+ return sum;
+ }
+
+}
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java?rev=1436556&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java Mon Jan 21 19:11:59 2013
@@ -0,0 +1,818 @@
+package org.apache.lucene.codecs.compressing;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.TermVectorsReader;
+import org.apache.lucene.codecs.TermVectorsWriter;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.MergeState;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.packed.BlockPackedWriter;
+import org.apache.lucene.util.packed.PackedInts;
+
+/**
+ * {@link TermVectorsWriter} for {@link CompressingTermVectorsFormat}.
+ * @lucene.experimental
+ */
+public final class CompressingTermVectorsWriter extends TermVectorsWriter {
+
+ static final String VECTORS_EXTENSION = "tvd";
+ static final String VECTORS_INDEX_EXTENSION = "tvx";
+
+ static final String CODEC_SFX_IDX = "Index";
+ static final String CODEC_SFX_DAT = "Data";
+
+ static final int VERSION_START = 0;
+ static final int VERSION_CURRENT = VERSION_START;
+
+ static final int BLOCK_SIZE = 64;
+
+ static final int POSITIONS = 0x01;
+ static final int OFFSETS = 0x02;
+ static final int PAYLOADS = 0x04;
+ static final int FLAGS_BITS = PackedInts.bitsRequired(POSITIONS | OFFSETS | PAYLOADS);
+
+ private final Directory directory;
+ private final String segment;
+ private final String segmentSuffix;
+ private CompressingStoredFieldsIndexWriter indexWriter;
+ private IndexOutput vectorsStream;
+
+ private final CompressionMode compressionMode;
+ private final Compressor compressor;
+ private final int chunkSize;
+
+ /** a pending doc */
+ private class DocData {
+ final int numFields;
+ final Deque<FieldData> fields;
+ final int posStart, offStart, payStart;
+ DocData(int numFields, int posStart, int offStart, int payStart) {
+ this.numFields = numFields;
+ this.fields = new ArrayDeque<FieldData>(numFields);
+ this.posStart = posStart;
+ this.offStart = offStart;
+ this.payStart = payStart;
+ }
+ FieldData addField(int fieldNum, int numTerms, boolean positions, boolean offsets, boolean payloads) {
+ final FieldData field;
+ if (fields.isEmpty()) {
+ field = new FieldData(fieldNum, numTerms, positions, offsets, payloads, posStart, offStart, payStart);
+ } else {
+ final FieldData last = fields.getLast();
+ final int posStart = last.posStart + (last.hasPositions ? last.totalPositions : 0);
+ final int offStart = last.offStart + (last.hasOffsets ? last.totalPositions : 0);
+ final int payStart = last.payStart + (last.hasPayloads ? last.totalPositions : 0);
+ field = new FieldData(fieldNum, numTerms, positions, offsets, payloads, posStart, offStart, payStart);
+ }
+ fields.add(field);
+ return field;
+ }
+ }
+
+ private DocData addDocData(int numVectorFields) {
+ FieldData last = null;
+ for (Iterator<DocData> it = pendingDocs.descendingIterator(); it.hasNext(); ) {
+ final DocData doc = it.next();
+ if (!doc.fields.isEmpty()) {
+ last = doc.fields.getLast();
+ break;
+ }
+ }
+ final DocData doc;
+ if (last == null) {
+ doc = new DocData(numVectorFields, 0, 0, 0);
+ } else {
+ final int posStart = last.posStart + (last.hasPositions ? last.totalPositions : 0);
+ final int offStart = last.offStart + (last.hasOffsets ? last.totalPositions : 0);
+ final int payStart = last.payStart + (last.hasPayloads ? last.totalPositions : 0);
+ doc = new DocData(numVectorFields, posStart, offStart, payStart);
+ }
+ pendingDocs.add(doc);
+ return doc;
+ }
+
+ /** a pending field */
+ private class FieldData {
+ final boolean hasPositions, hasOffsets, hasPayloads;
+ final int fieldNum, flags, numTerms;
+ final int[] freqs, prefixLengths, suffixLengths;
+ final int posStart, offStart, payStart;
+ int totalPositions;
+ int ord;
+ FieldData(int fieldNum, int numTerms, boolean positions, boolean offsets, boolean payloads,
+ int posStart, int offStart, int payStart) {
+ this.fieldNum = fieldNum;
+ this.numTerms = numTerms;
+ this.hasPositions = positions;
+ this.hasOffsets = offsets;
+ this.hasPayloads = payloads;
+ this.flags = (positions ? POSITIONS : 0) | (offsets ? OFFSETS : 0) | (payloads ? PAYLOADS : 0);
+ this.freqs = new int[numTerms];
+ this.prefixLengths = new int[numTerms];
+ this.suffixLengths = new int[numTerms];
+ this.posStart = posStart;
+ this.offStart = offStart;
+ this.payStart = payStart;
+ totalPositions = 0;
+ ord = 0;
+ }
+ void addTerm(int freq, int prefixLength, int suffixLength) {
+ freqs[ord] = freq;
+ prefixLengths[ord] = prefixLength;
+ suffixLengths[ord] = suffixLength;
+ ++ord;
+ }
+ void addPosition(int position, int startOffset, int length, int payloadLength) {
+ if (hasPositions) {
+ if (posStart + totalPositions == positionsBuf.length) {
+ positionsBuf = ArrayUtil.grow(positionsBuf);
+ }
+ positionsBuf[posStart + totalPositions] = position;
+ }
+ if (hasOffsets) {
+ if (offStart + totalPositions == startOffsetsBuf.length) {
+ final int newLength = ArrayUtil.oversize(offStart + totalPositions, 4);
+ startOffsetsBuf = Arrays.copyOf(startOffsetsBuf, newLength);
+ lengthsBuf = Arrays.copyOf(lengthsBuf, newLength);
+ }
+ startOffsetsBuf[offStart + totalPositions] = startOffset;
+ lengthsBuf[offStart + totalPositions] = length;
+ }
+ if (hasPayloads) {
+ if (payStart + totalPositions == payloadLengthsBuf.length) {
+ payloadLengthsBuf = ArrayUtil.grow(payloadLengthsBuf);
+ }
+ payloadLengthsBuf[payStart + totalPositions] = payloadLength;
+ }
+ ++totalPositions;
+ }
+ }
+
+ private int numDocs; // total number of docs seen
+ private final Deque<DocData> pendingDocs; // pending docs
+ private DocData curDoc; // current document
+ private FieldData curField; // current field
+ private final BytesRef lastTerm;
+ private int[] positionsBuf, startOffsetsBuf, lengthsBuf, payloadLengthsBuf;
+ private final GrowableByteArrayDataOutput termSuffixes; // buffered term suffixes
+ private final GrowableByteArrayDataOutput payloadBytes; // buffered term payloads
+ private final BlockPackedWriter writer;
+
+ /** Sole constructor. */
+ public CompressingTermVectorsWriter(Directory directory, SegmentInfo si, String segmentSuffix, IOContext context,
+ String formatName, CompressionMode compressionMode, int chunkSize) throws IOException {
+ assert directory != null;
+ this.directory = directory;
+ this.segment = si.name;
+ this.segmentSuffix = segmentSuffix;
+ this.compressionMode = compressionMode;
+ this.compressor = compressionMode.newCompressor();
+ this.chunkSize = chunkSize;
+
+ numDocs = 0;
+ pendingDocs = new ArrayDeque<DocData>();
+ termSuffixes = new GrowableByteArrayDataOutput(ArrayUtil.oversize(chunkSize, 1));
+ payloadBytes = new GrowableByteArrayDataOutput(ArrayUtil.oversize(1, 1));
+ lastTerm = new BytesRef(ArrayUtil.oversize(30, 1));
+
+ boolean success = false;
+ IndexOutput indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_INDEX_EXTENSION), context);
+ try {
+ vectorsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION), context);
+
+ final String codecNameIdx = formatName + CODEC_SFX_IDX;
+ final String codecNameDat = formatName + CODEC_SFX_DAT;
+ CodecUtil.writeHeader(indexStream, codecNameIdx, VERSION_CURRENT);
+ CodecUtil.writeHeader(vectorsStream, codecNameDat, VERSION_CURRENT);
+ assert CodecUtil.headerLength(codecNameDat) == vectorsStream.getFilePointer();
+ assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer();
+
+ indexWriter = new CompressingStoredFieldsIndexWriter(indexStream);
+ indexStream = null;
+
+ vectorsStream.writeVInt(PackedInts.VERSION_CURRENT);
+ vectorsStream.writeVInt(chunkSize);
+ writer = new BlockPackedWriter(vectorsStream, BLOCK_SIZE);
+
+ positionsBuf = new int[1024];
+ startOffsetsBuf = new int[1024];
+ lengthsBuf = new int[1024];
+ payloadLengthsBuf = new int[1024];
+
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(indexStream);
+ abort();
+ }
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ IOUtils.close(vectorsStream, indexWriter);
+ } finally {
+ vectorsStream = null;
+ indexWriter = null;
+ }
+ }
+
+ @Override
+ public void abort() {
+ IOUtils.closeWhileHandlingException(this);
+ IOUtils.deleteFilesIgnoringExceptions(directory,
+ IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION),
+ IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_INDEX_EXTENSION));
+ }
+
+ @Override
+ public void startDocument(int numVectorFields) throws IOException {
+ curDoc = addDocData(numVectorFields);
+ }
+
+ @Override
+ public void finishDocument() throws IOException {
+ // append the payload bytes of the doc after its terms
+ termSuffixes.writeBytes(payloadBytes.bytes, payloadBytes.length);
+ payloadBytes.length = 0;
+ ++numDocs;
+ if (triggerFlush()) {
+ flush();
+ }
+ curDoc = null;
+ }
+
+ @Override
+ public void startField(FieldInfo info, int numTerms, boolean positions,
+ boolean offsets, boolean payloads) throws IOException {
+ curField = curDoc.addField(info.number, numTerms, positions, offsets, payloads);
+ lastTerm.length = 0;
+ }
+
+ @Override
+ public void finishField() throws IOException {
+ curField = null;
+ }
+
+ @Override
+ public void startTerm(BytesRef term, int freq) throws IOException {
+ assert freq >= 1;
+ final int prefix = StringHelper.bytesDifference(lastTerm, term);
+ curField.addTerm(freq, prefix, term.length - prefix);
+ termSuffixes.writeBytes(term.bytes, term.offset + prefix, term.length - prefix);
+ // copy last term
+ if (lastTerm.bytes.length < term.length) {
+ lastTerm.bytes = new byte[ArrayUtil.oversize(term.length, 1)];
+ }
+ lastTerm.offset = 0;
+ lastTerm.length = term.length;
+ System.arraycopy(term.bytes, term.offset, lastTerm.bytes, 0, term.length);
+ }
+
+ @Override
+ public void addPosition(int position, int startOffset, int endOffset,
+ BytesRef payload) throws IOException {
+ assert curField.flags != 0;
+ curField.addPosition(position, startOffset, endOffset - startOffset, payload == null ? 0 : payload.length);
+ if (curField.hasPayloads && payload != null) {
+ payloadBytes.writeBytes(payload.bytes, payload.offset, payload.length);
+ }
+ }
+
+ private boolean triggerFlush() {
+ return termSuffixes.length >= chunkSize || pendingDocs.size() >= chunkSize;
+ }
+
+ private void flush() throws IOException {
+ final int chunkDocs = pendingDocs.size();
+ assert chunkDocs > 0 : chunkDocs;
+
+ // write the index file
+ indexWriter.writeIndex(chunkDocs, vectorsStream.getFilePointer());
+
+ final int docBase = numDocs - chunkDocs;
+ vectorsStream.writeVInt(docBase);
+ vectorsStream.writeVInt(chunkDocs);
+
+ // total number of fields of the chunk
+ final int totalFields = flushNumFields(chunkDocs);
+
+ if (totalFields > 0) {
+ // unique field numbers (sorted)
+ final int[] fieldNums = flushFieldNums();
+ // offsets in the array of unique field numbers
+ flushFields(totalFields, fieldNums);
+ // flags (does the field have positions, offsets, payloads?)
+ flushFlags(totalFields, fieldNums);
+ // number of terms of each field
+ flushNumTerms(totalFields);
+ // prefix and suffix lengths for each field
+ flushTermLengths();
+ // term freqs - 1 (because termFreq is always >=1) for each term
+ flushTermFreqs();
+ // positions for all terms, when enabled
+ flushPositions();
+ // offsets for all terms, when enabled
+ flushOffsets(fieldNums);
+ // payload lengths for all terms, when enabled
+ flushPayloadLengths();
+
+ // compress terms and payloads and write them to the output
+ compressor.compress(termSuffixes.bytes, 0, termSuffixes.length, vectorsStream);
+ }
+
+ // reset
+ pendingDocs.clear();
+ curDoc = null;
+ curField = null;
+ termSuffixes.length = 0;
+ }
+
+ private int flushNumFields(int chunkDocs) throws IOException {
+ if (chunkDocs == 1) {
+ final int numFields = pendingDocs.getFirst().numFields;
+ vectorsStream.writeVInt(numFields);
+ return numFields;
+ } else {
+ writer.reset(vectorsStream);
+ int totalFields = 0;
+ for (DocData dd : pendingDocs) {
+ writer.add(dd.numFields);
+ totalFields += dd.numFields;
+ }
+ writer.finish();
+ return totalFields;
+ }
+ }
+
+ /** Returns a sorted array containing unique field numbers */
+ private int[] flushFieldNums() throws IOException {
+ SortedSet<Integer> fieldNums = new TreeSet<Integer>();
+ for (DocData dd : pendingDocs) {
+ for (FieldData fd : dd.fields) {
+ fieldNums.add(fd.fieldNum);
+ }
+ }
+
+ final int numDistinctFields = fieldNums.size();
+ assert numDistinctFields > 0;
+ final int bitsRequired = PackedInts.bitsRequired(fieldNums.last());
+ final int token = (Math.min(numDistinctFields - 1, 0x07) << 5) | bitsRequired;
+ vectorsStream.writeByte((byte) token);
+ if (numDistinctFields - 1 >= 0x07) {
+ vectorsStream.writeVInt(numDistinctFields - 1 - 0x07);
+ }
+ final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, fieldNums.size(), bitsRequired, 1);
+ for (Integer fieldNum : fieldNums) {
+ writer.add(fieldNum);
+ }
+ writer.finish();
+
+ int[] fns = new int[fieldNums.size()];
+ int i = 0;
+ for (Integer key : fieldNums) {
+ fns[i++] = key;
+ }
+ return fns;
+ }
+
+ private void flushFields(int totalFields, int[] fieldNums) throws IOException {
+ final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, totalFields, PackedInts.bitsRequired(fieldNums.length - 1), 1);
+ for (DocData dd : pendingDocs) {
+ for (FieldData fd : dd.fields) {
+ final int fieldNumIndex = Arrays.binarySearch(fieldNums, fd.fieldNum);
+ assert fieldNumIndex >= 0;
+ writer.add(fieldNumIndex);
+ }
+ }
+ writer.finish();
+ }
+
+ private void flushFlags(int totalFields, int[] fieldNums) throws IOException {
+ // check if fields always have the same flags
+ boolean nonChangingFlags = true;
+ int[] fieldFlags = new int[fieldNums.length];
+ Arrays.fill(fieldFlags, -1);
+ outer:
+ for (DocData dd : pendingDocs) {
+ for (FieldData fd : dd.fields) {
+ final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum);
+ assert fieldNumOff >= 0;
+ if (fieldFlags[fieldNumOff] == -1) {
+ fieldFlags[fieldNumOff] = fd.flags;
+ } else if (fieldFlags[fieldNumOff] != fd.flags) {
+ nonChangingFlags = false;
+ break outer;
+ }
+ }
+ }
+
+ if (nonChangingFlags) {
+ // write one flag per field num
+ vectorsStream.writeVInt(0);
+ final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, fieldFlags.length, FLAGS_BITS, 1);
+ for (int flags : fieldFlags) {
+ assert flags >= 0;
+ writer.add(flags);
+ }
+ assert writer.ord() == fieldFlags.length - 1;
+ writer.finish();
+ } else {
+ // write one flag for every field instance
+ vectorsStream.writeVInt(1);
+ final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, totalFields, FLAGS_BITS, 1);
+ for (DocData dd : pendingDocs) {
+ for (FieldData fd : dd.fields) {
+ writer.add(fd.flags);
+ }
+ }
+ assert writer.ord() == totalFields - 1;
+ writer.finish();
+ }
+ }
+
+ private void flushNumTerms(int totalFields) throws IOException {
+ int maxNumTerms = 0;
+ for (DocData dd : pendingDocs) {
+ for (FieldData fd : dd.fields) {
+ maxNumTerms |= fd.numTerms;
+ }
+ }
+ final int bitsRequired = PackedInts.bitsRequired(maxNumTerms);
+ vectorsStream.writeVInt(bitsRequired);
+ final PackedInts.Writer writer = PackedInts.getWriterNoHeader(
+ vectorsStream, PackedInts.Format.PACKED, totalFields, bitsRequired, 1);
+ for (DocData dd : pendingDocs) {
+ for (FieldData fd : dd.fields) {
+ writer.add(fd.numTerms);
+ }
+ }
+ assert writer.ord() == totalFields - 1;
+ writer.finish();
+ }
+
+ private void flushTermLengths() throws IOException {
+ writer.reset(vectorsStream);
+ for (DocData dd : pendingDocs) {
+ for (FieldData fd : dd.fields) {
+ for (int i = 0; i < fd.numTerms; ++i) {
+ writer.add(fd.prefixLengths[i]);
+ }
+ }
+ }
+ writer.finish();
+ writer.reset(vectorsStream);
+ for (DocData dd : pendingDocs) {
+ for (FieldData fd : dd.fields) {
+ for (int i = 0; i < fd.numTerms; ++i) {
+ writer.add(fd.suffixLengths[i]);
+ }
+ }
+ }
+ writer.finish();
+ }
+
+ private void flushTermFreqs() throws IOException {
+ writer.reset(vectorsStream);
+ for (DocData dd : pendingDocs) {
+ for (FieldData fd : dd.fields) {
+ for (int i = 0; i < fd.numTerms; ++i) {
+ writer.add(fd.freqs[i] - 1);
+ }
+ }
+ }
+ writer.finish();
+ }
+
+ private void flushPositions() throws IOException {
+ writer.reset(vectorsStream);
+ for (DocData dd : pendingDocs) {
+ for (FieldData fd : dd.fields) {
+ if (fd.hasPositions) {
+ int pos = 0;
+ for (int i = 0; i < fd.numTerms; ++i) {
+ int previousPosition = 0;
+ for (int j = 0; j < fd.freqs[i]; ++j) {
+ final int position = positionsBuf[fd .posStart + pos++];
+ writer.add(position - previousPosition);
+ previousPosition = position;
+ }
+ }
+ assert pos == fd.totalPositions;
+ }
+ }
+ }
+ writer.finish();
+ }
+
+ private void flushOffsets(int[] fieldNums) throws IOException {
+ boolean hasOffsets = false;
+ long[] sumPos = new long[fieldNums.length];
+ long[] sumOffsets = new long[fieldNums.length];
+ for (DocData dd : pendingDocs) {
+ for (FieldData fd : dd.fields) {
+ hasOffsets |= fd.hasOffsets;
+ if (fd.hasOffsets && fd.hasPositions) {
+ final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum);
+ int pos = 0;
+ for (int i = 0; i < fd.numTerms; ++i) {
+ int previousPos = 0;
+ int previousOff = 0;
+ for (int j = 0; j < fd.freqs[i]; ++j) {
+ final int position = positionsBuf[fd.posStart + pos];
+ final int startOffset = startOffsetsBuf[fd.offStart + pos];
+ sumPos[fieldNumOff] += position - previousPos;
+ sumOffsets[fieldNumOff] += startOffset - previousOff;
+ previousPos = position;
+ previousOff = startOffset;
+ ++pos;
+ }
+ }
+ assert pos == fd.totalPositions;
+ }
+ }
+ }
+
+ if (!hasOffsets) {
+ // nothing to do
+ return;
+ }
+
+ final float[] charsPerTerm = new float[fieldNums.length];
+ for (int i = 0; i < fieldNums.length; ++i) {
+ charsPerTerm[i] = (sumPos[i] <= 0 || sumOffsets[i] <= 0) ? 0 : (float) ((double) sumOffsets[i] / sumPos[i]);
+ }
+
+ // start offsets
+ for (int i = 0; i < fieldNums.length; ++i) {
+ vectorsStream.writeInt(Float.floatToRawIntBits(charsPerTerm[i]));
+ }
+
+ writer.reset(vectorsStream);
+ for (DocData dd : pendingDocs) {
+ for (FieldData fd : dd.fields) {
+ if ((fd.flags & OFFSETS) != 0) {
+ final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum);
+ final float cpt = charsPerTerm[fieldNumOff];
+ int pos = 0;
+ for (int i = 0; i < fd.numTerms; ++i) {
+ int previousPos = 0;
+ int previousOff = 0;
+ for (int j = 0; j < fd.freqs[i]; ++j) {
+ final int position = fd.hasPositions ? positionsBuf[fd.posStart + pos] : 0;
+ final int startOffset = startOffsetsBuf[fd.offStart + pos];
+ writer.add(startOffset - previousOff - (int) (cpt * (position - previousPos)));
+ previousPos = position;
+ previousOff = startOffset;
+ ++pos;
+ }
+ }
+ }
+ }
+ }
+ writer.finish();
+
+ // lengths
+ writer.reset(vectorsStream);
+ for (DocData dd : pendingDocs) {
+ for (FieldData fd : dd.fields) {
+ if ((fd.flags & OFFSETS) != 0) {
+ int pos = 0;
+ for (int i = 0; i < fd.numTerms; ++i) {
+ for (int j = 0; j < fd.freqs[i]; ++j) {
+ writer.add(lengthsBuf[fd.offStart + pos++] - fd.prefixLengths[i] - fd.suffixLengths[i]);
+ }
+ }
+ assert pos == fd.totalPositions;
+ }
+ }
+ }
+ writer.finish();
+ }
+
+ private void flushPayloadLengths() throws IOException {
+ writer.reset(vectorsStream);
+ for (DocData dd : pendingDocs) {
+ for (FieldData fd : dd.fields) {
+ if (fd.hasPayloads) {
+ for (int i = 0; i < fd.totalPositions; ++i) {
+ writer.add(payloadLengthsBuf[fd.payStart + i]);
+ }
+ }
+ }
+ }
+ writer.finish();
+ }
+
+ @Override
+ public void finish(FieldInfos fis, int numDocs) throws IOException {
+ if (!pendingDocs.isEmpty()) {
+ flush();
+ }
+ if (numDocs != this.numDocs) {
+ throw new RuntimeException("Wrote " + this.numDocs + " docs, finish called with numDocs=" + numDocs);
+ }
+ indexWriter.finish(numDocs);
+ }
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ }
+
+ @Override
+ public void addProx(int numProx, DataInput positions, DataInput offsets)
+ throws IOException {
+ assert (curField.hasPositions) == (positions != null);
+ assert (curField.hasOffsets) == (offsets != null);
+
+ if (curField.hasPositions) {
+ final int posStart = curField.posStart + curField.totalPositions;
+ if (posStart + numProx > positionsBuf.length) {
+ positionsBuf = ArrayUtil.grow(positionsBuf, posStart + numProx);
+ }
+ int position = 0;
+ if (curField.hasPayloads) {
+ final int payStart = curField.payStart + curField.totalPositions;
+ if (payStart + numProx > payloadLengthsBuf.length) {
+ payloadLengthsBuf = ArrayUtil.grow(payloadLengthsBuf, payStart + numProx);
+ }
+ for (int i = 0; i < numProx; ++i) {
+ final int code = positions.readVInt();
+ if ((code & 1) != 0) {
+ // This position has a payload
+ final int payloadLength = positions.readVInt();
+ payloadLengthsBuf[payStart + i] = payloadLength;
+ payloadBytes.copyBytes(positions, payloadLength);
+ } else {
+ payloadLengthsBuf[payStart + i] = 0;
+ }
+ position += code >>> 1;
+ positionsBuf[posStart + i] = position;
+ }
+ } else {
+ for (int i = 0; i < numProx; ++i) {
+ position += (positions.readVInt() >>> 1);
+ positionsBuf[posStart + i] = position;
+ }
+ }
+ }
+
+ if (curField.hasOffsets) {
+ final int offStart = curField.offStart + curField.totalPositions;
+ if (offStart + numProx > startOffsetsBuf.length) {
+ final int newLength = ArrayUtil.oversize(offStart + numProx, 4);
+ startOffsetsBuf = Arrays.copyOf(startOffsetsBuf, newLength);
+ lengthsBuf = Arrays.copyOf(lengthsBuf, newLength);
+ }
+ int lastOffset = 0, startOffset, endOffset;
+ for (int i = 0; i < numProx; ++i) {
+ startOffset = lastOffset + offsets.readVInt();
+ endOffset = startOffset + offsets.readVInt();
+ lastOffset = endOffset;
+ startOffsetsBuf[offStart + i] = startOffset;
+ lengthsBuf[offStart + i] = endOffset - startOffset;
+ }
+ }
+
+ curField.totalPositions += numProx;
+ }
+
+ @Override
+ public int merge(MergeState mergeState) throws IOException {
+ int docCount = 0;
+ int idx = 0;
+
+ for (AtomicReader reader : mergeState.readers) {
+ final SegmentReader matchingSegmentReader = mergeState.matchingSegmentReaders[idx++];
+ CompressingTermVectorsReader matchingVectorsReader = null;
+ if (matchingSegmentReader != null) {
+ final TermVectorsReader vectorsReader = matchingSegmentReader.getTermVectorsReader();
+ // we can only bulk-copy if the matching reader is also a CompressingTermVectorsReader
+ if (vectorsReader != null && vectorsReader instanceof CompressingTermVectorsReader) {
+ matchingVectorsReader = (CompressingTermVectorsReader) vectorsReader;
+ }
+ }
+
+ final int maxDoc = reader.maxDoc();
+ final Bits liveDocs = reader.getLiveDocs();
+
+ if (matchingVectorsReader == null
+ || matchingVectorsReader.getCompressionMode() != compressionMode
+ || matchingVectorsReader.getChunkSize() != chunkSize
+ || matchingVectorsReader.getPackedIntsVersion() != PackedInts.VERSION_CURRENT) {
+ // naive merge...
+ for (int i = nextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; i = nextLiveDoc(i + 1, liveDocs, maxDoc)) {
+ final Fields vectors = reader.getTermVectors(i);
+ addAllDocVectors(vectors, mergeState);
+ ++docCount;
+ mergeState.checkAbort.work(300);
+ }
+ } else {
+ final CompressingStoredFieldsIndexReader index = matchingVectorsReader.getIndex();
+ final IndexInput vectorsStream = matchingVectorsReader.getVectorsStream();
+ for (int i = nextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; ) {
+ if (pendingDocs.isEmpty()
+ && (i == 0 || index.getStartPointer(i - 1) < index.getStartPointer(i))) { // start of a chunk
+ final long startPointer = index.getStartPointer(i);
+ vectorsStream.seek(startPointer);
+ final int docBase = vectorsStream.readVInt();
+ final int chunkDocs = vectorsStream.readVInt();
+ assert docBase + chunkDocs <= matchingSegmentReader.maxDoc();
+ if (docBase + chunkDocs < matchingSegmentReader.maxDoc()
+ && nextDeletedDoc(docBase, liveDocs, docBase + chunkDocs) == docBase + chunkDocs) {
+ final long chunkEnd = index.getStartPointer(docBase + chunkDocs);
+ final long chunkLength = chunkEnd - vectorsStream.getFilePointer();
+ indexWriter.writeIndex(chunkDocs, this.vectorsStream.getFilePointer());
+ this.vectorsStream.writeVInt(docCount);
+ this.vectorsStream.writeVInt(chunkDocs);
+ this.vectorsStream.copyBytes(vectorsStream, chunkLength);
+ docCount += chunkDocs;
+ this.numDocs += chunkDocs;
+ mergeState.checkAbort.work(300 * chunkDocs);
+ i = nextLiveDoc(docBase + chunkDocs, liveDocs, maxDoc);
+ } else {
+ for (; i < docBase + chunkDocs; i = nextLiveDoc(i + 1, liveDocs, maxDoc)) {
+ final Fields vectors = reader.getTermVectors(i);
+ addAllDocVectors(vectors, mergeState);
+ ++docCount;
+ mergeState.checkAbort.work(300);
+ }
+ }
+ } else {
+ final Fields vectors = reader.getTermVectors(i);
+ addAllDocVectors(vectors, mergeState);
+ ++docCount;
+ mergeState.checkAbort.work(300);
+ i = nextLiveDoc(i + 1, liveDocs, maxDoc);
+ }
+ }
+ }
+ }
+ finish(mergeState.fieldInfos, docCount);
+ return docCount;
+ }
+
+ private static int nextLiveDoc(int doc, Bits liveDocs, int maxDoc) {
+ if (liveDocs == null) {
+ return doc;
+ }
+ while (doc < maxDoc && !liveDocs.get(doc)) {
+ ++doc;
+ }
+ return doc;
+ }
+
+ private static int nextDeletedDoc(int doc, Bits liveDocs, int maxDoc) {
+ if (liveDocs == null) {
+ return maxDoc;
+ }
+ while (doc < maxDoc && liveDocs.get(doc)) {
+ ++doc;
+ }
+ return doc;
+ }
+
+}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java?rev=1436556&r1=1436555&r2=1436556&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java Mon Jan 21 19:11:59 2013
@@ -71,11 +71,12 @@ public final class BlockPackedReader {
return i;
}
- final DataInput in;
+ DataInput in;
final int packedIntsVersion;
- final long valueCount;
+ long valueCount;
final int blockSize;
- final LongsRef values;
+ final long[] values;
+ final LongsRef valuesRef;
byte[] blocks;
int off;
long ord;
@@ -87,10 +88,17 @@ public final class BlockPackedReader {
*/
public BlockPackedReader(DataInput in, int packedIntsVersion, int blockSize, long valueCount) {
checkBlockSize(blockSize);
- this.in = in;
this.packedIntsVersion = packedIntsVersion;
this.blockSize = blockSize;
- this.values = new LongsRef(blockSize);
+ this.values = new long[blockSize];
+ this.valuesRef = new LongsRef(this.values, 0, 0);
+ reset(in, valueCount);
+ }
+
+ /** Reset the current reader to wrap a stream of <code>valueCount</code>
+ * values contained in <code>in</code>. The block size remains unchanged. */
+ public void reset(DataInput in, long valueCount) {
+ this.in = in;
assert valueCount >= 0;
this.valueCount = valueCount;
off = blockSize;
@@ -159,9 +167,15 @@ public final class BlockPackedReader {
/** Read the next value. */
public long next() throws IOException {
- next(1);
- assert values.length == 1;
- return values.longs[values.offset];
+ if (ord == valueCount) {
+ throw new EOFException();
+ }
+ if (off == blockSize) {
+ refill();
+ }
+ final long value = values[off++];
+ ++ord;
+ return value;
}
/** Read between <tt>1</tt> and <code>count</code> values. */
@@ -177,11 +191,11 @@ public final class BlockPackedReader {
count = Math.min(count, blockSize - off);
count = (int) Math.min(count, valueCount - ord);
- values.offset = off;
- values.length = count;
+ valuesRef.offset = off;
+ valuesRef.length = count;
off += count;
ord += count;
- return values;
+ return valuesRef;
}
private void refill() throws IOException {
@@ -195,7 +209,7 @@ public final class BlockPackedReader {
assert minEquals0 || minValue != 0;
if (bitsPerValue == 0) {
- Arrays.fill(values.longs, minValue);
+ Arrays.fill(values, minValue);
} else {
final PackedInts.Decoder decoder = PackedInts.getDecoder(PackedInts.Format.PACKED, packedIntsVersion, bitsPerValue);
final int iterations = blockSize / decoder.valueCount();
@@ -208,11 +222,11 @@ public final class BlockPackedReader {
final int blocksCount = (int) PackedInts.Format.PACKED.byteCount(packedIntsVersion, valueCount, bitsPerValue);
in.readBytes(blocks, 0, blocksCount);
- decoder.decode(blocks, 0, values.longs, 0, iterations);
+ decoder.decode(blocks, 0, values, 0, iterations);
if (minValue != 0) {
for (int i = 0; i < valueCount; ++i) {
- values.longs[i] += minValue;
+ values[i] += minValue;
}
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java?rev=1436556&r1=1436555&r2=1436556&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java Mon Jan 21 19:11:59 2013
@@ -62,7 +62,7 @@ public final class BlockPackedWriter {
out.writeByte((byte) i);
}
- final DataOutput out;
+ DataOutput out;
final long[] values;
byte[] blocks;
int off;
@@ -75,8 +75,14 @@ public final class BlockPackedWriter {
*/
public BlockPackedWriter(DataOutput out, int blockSize) {
checkBlockSize(blockSize);
- this.out = out;
+ reset(out);
values = new long[blockSize];
+ }
+
+ /** Reset this writer to wrap <code>out</code>. The block size remains unchanged. */
+ public void reset(DataOutput out) {
+ assert out != null;
+ this.out = out;
off = 0;
ord = 0L;
finished = false;
@@ -99,7 +105,8 @@ public final class BlockPackedWriter {
}
/** Flush all buffered data to disk. This instance is not usable anymore
- * after this method has been called. */
+ * after this method has been called until {@link #reset(DataOutput)} has
+ * been called. */
public void finish() throws IOException {
checkNotFinished();
if (off > 0) {
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1436556&r1=1436555&r2=1436556&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Mon Jan 21 19:11:59 2013
@@ -1482,7 +1482,15 @@ public class TestIndexWriter extends Luc
doc.add(newField("c", "val", customType));
writer.addDocument(doc);
// Adding just one document does not call flush yet.
- assertEquals("only the stored and term vector files should exist in the directory", 5 + extraFileCount, dir.listAll().length);
+ int computedExtraFileCount = 0;
+ for (String file : dir.listAll()) {
+ if (file.lastIndexOf('.') < 0
+ // don't count stored fields and term vectors in
+ || !Arrays.asList("fdx", "fdt", "tvx", "tvd", "tvf").contains(file.substring(file.lastIndexOf('.') + 1))) {
+ ++computedExtraFileCount;
+ }
+ }
+ assertEquals("only the stored and term vector files should exist in the directory", extraFileCount, computedExtraFileCount);
doc = new Document();
doc.add(newField("c", "val", customType));