You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2013/01/22 01:31:37 UTC
svn commit: r1436700 - in
/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv:
DiskDocValuesConsumer.java DiskDocValuesProducer.java
Author: jpountz
Date: Tue Jan 22 00:31:37 2013
New Revision: 1436700
URL: http://svn.apache.org/viewvc?rev=1436700&view=rev
Log:
Same for DiskDocValues (using direct readers and larger blocks).
Modified:
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java?rev=1436700&r1=1436699&r2=1436700&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java Tue Jan 22 00:31:37 2013
@@ -28,10 +28,13 @@ import org.apache.lucene.index.SegmentWr
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.packed.BlockPackedWriter;
import org.apache.lucene.util.packed.PackedInts;
-import org.apache.lucene.util.packed.PackedInts.FormatAndBits;
class DiskDocValuesConsumer extends DocValuesConsumer {
+
+ static final int BLOCK_SIZE = 16384;
+
final IndexOutput data, meta;
final int maxDoc;
@@ -57,38 +60,20 @@ class DiskDocValuesConsumer extends DocV
@Override
public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
- meta.writeVInt(field.number);
- long minValue = Long.MAX_VALUE;
- long maxValue = Long.MIN_VALUE;
int count = 0;
- for(Number nv : values) {
- long v = nv.longValue();
- minValue = Math.min(minValue, v);
- maxValue = Math.max(maxValue, v);
- count++;
+ for (@SuppressWarnings("unused") Number nv : values) {
+ ++count;
}
- meta.writeLong(minValue);
- long delta = maxValue - minValue;
- final int bitsPerValue;
- if (delta < 0) {
- bitsPerValue = 64;
- } else {
- bitsPerValue = PackedInts.bitsRequired(delta);
- }
- FormatAndBits formatAndBits = PackedInts.fastestFormatAndBits(count, bitsPerValue, PackedInts.COMPACT);
-
- // nocommit: refactor this crap in PackedInts.java
- // e.g. Header.load()/save() or something rather than how it works now.
- CodecUtil.writeHeader(meta, PackedInts.CODEC_NAME, PackedInts.VERSION_CURRENT);
- meta.writeVInt(bitsPerValue);
- meta.writeVInt(count);
- meta.writeVInt(formatAndBits.format.getId());
-
+
+ meta.writeVInt(field.number);
+ meta.writeVInt(PackedInts.VERSION_CURRENT);
meta.writeLong(data.getFilePointer());
-
- final PackedInts.Writer writer = PackedInts.getWriterNoHeader(data, formatAndBits.format, count, formatAndBits.bitsPerValue, 0);
- for(Number nv : values) {
- writer.add(nv.longValue() - minValue);
+ meta.writeVInt(count);
+ meta.writeVInt(BLOCK_SIZE);
+
+ final BlockPackedWriter writer = new BlockPackedWriter(data, BLOCK_SIZE);
+ for (Number nv : values) {
+ writer.add(nv.longValue());
}
writer.finish();
}
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java?rev=1436700&r1=1436699&r2=1436700&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java Tue Jan 22 00:31:37 2013
@@ -35,7 +35,7 @@ import org.apache.lucene.index.SortedDoc
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.BlockPackedReader;
class DiskDocValuesProducer extends DocValuesProducer {
private final Map<Integer,NumericEntry> numerics;
@@ -110,9 +110,10 @@ class DiskDocValuesProducer extends DocV
static NumericEntry readNumericEntry(IndexInput meta) throws IOException {
NumericEntry entry = new NumericEntry();
- entry.minValue = meta.readLong();
- entry.header = PackedInts.readHeader(meta);
+ entry.packedIntsVersion = meta.readVInt();
entry.offset = meta.readLong();
+ entry.count = meta.readVInt();
+ entry.blockSize = meta.readVInt();
return entry;
}
@@ -134,11 +135,12 @@ class DiskDocValuesProducer extends DocV
private NumericDocValues getNumeric(FieldInfo field, final NumericEntry entry) throws IOException {
final IndexInput data = this.data.clone();
data.seek(entry.offset);
- final PackedInts.Reader reader = PackedInts.getDirectReaderNoHeader(data, entry.header);
+
+ final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true);
return new NumericDocValues() {
@Override
public long get(int docID) {
- return entry.minValue + reader.get(docID);
+ return reader.get(docID);
}
};
}
@@ -229,9 +231,10 @@ class DiskDocValuesProducer extends DocV
static class NumericEntry {
long offset;
-
- long minValue;
- PackedInts.Header header;
+
+ int packedIntsVersion;
+ int count;
+ int blockSize;
}
static class BinaryEntry {