You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2018/05/08 08:34:46 UTC
[3/3] lucene-solr:master: LUCENE-8267: removed references to memory
codecs.
LUCENE-8267: removed references to memory codecs.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/85c00e77
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/85c00e77
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/85c00e77
Branch: refs/heads/master
Commit: 85c00e77efdf53f30da6eaffd38c2b016a7805bc
Parents: 08ee037
Author: Dawid Weiss <dw...@apache.org>
Authored: Tue May 8 10:32:11 2018 +0200
Committer: Dawid Weiss <dw...@apache.org>
Committed: Tue May 8 10:32:11 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
lucene/MIGRATE.txt | 4 +
.../codecs/memory/MemoryDocValuesConsumer.java | 539 ---------
.../codecs/memory/MemoryDocValuesFormat.java | 72 --
.../codecs/memory/MemoryDocValuesProducer.java | 877 ---------------
.../codecs/memory/MemoryPostingsFormat.java | 1037 ------------------
.../org.apache.lucene.codecs.DocValuesFormat | 1 -
.../org.apache.lucene.codecs.PostingsFormat | 2 +-
.../memory/TestMemoryDocValuesFormat.java | 39 -
.../codecs/memory/TestMemoryPostingsFormat.java | 35 -
.../org/apache/lucene/TestExternalCodecs.java | 5 +-
.../perfield/TestPerFieldDocValuesFormat.java | 2 +-
.../perfield/TestPerFieldPostingsFormat2.java | 24 +-
.../lucene/index/Test2BBinaryDocValues.java | 2 +-
.../org/apache/lucene/index/Test2BDocs.java | 2 +-
.../lucene/index/Test2BNumericDocValues.java | 2 +-
.../org/apache/lucene/index/Test2BPoints.java | 2 +-
.../apache/lucene/index/Test2BPositions.java | 2 +-
.../org/apache/lucene/index/Test2BPostings.java | 2 +-
.../lucene/index/Test2BPostingsBytes.java | 2 +-
.../index/Test2BSortedDocValuesFixedSorted.java | 2 +-
.../lucene/index/Test2BSortedDocValuesOrds.java | 2 +-
.../org/apache/lucene/index/Test2BTerms.java | 2 +-
.../org/apache/lucene/index/TestAddIndexes.java | 7 +-
.../apache/lucene/index/TestBagOfPositions.java | 2 +-
.../apache/lucene/index/TestBagOfPostings.java | 2 +-
.../lucene/index/TestDuelingCodecsAtNight.java | 2 +-
.../lucene/index/TestIndexWriterCommit.java | 1 -
.../lucene/index/TestIndexWriterDelete.java | 1 -
.../lucene/index/TestIndexWriterMaxDocs.java | 2 +-
.../lucene/index/TestIndexWriterOnDiskFull.java | 1 -
.../lucene/index/TestLazyProxSkipping.java | 1 -
.../apache/lucene/index/TestLongPostings.java | 2 +-
.../org/apache/lucene/index/TestNRTThreads.java | 2 +-
.../test/org/apache/lucene/index/TestNorms.java | 2 +-
.../apache/lucene/index/TestRollingUpdates.java | 6 +-
.../org/apache/lucene/index/TestTermsEnum.java | 2 +-
.../TestControlledRealTimeReopenThread.java | 2 +-
.../lucene/search/TestSearchWithThreads.java | 2 +-
.../lucene/search/TestSearcherManager.java | 2 +-
.../lucene/search/TestShardSearching.java | 2 +-
.../lucene/search/TestSortedSetSelector.java | 2 +-
.../org/apache/lucene/util/fst/TestFSTs.java | 2 +-
.../search/TestInetAddressRangeQueries.java | 2 +-
.../lucene/replicator/nrt/SimpleServer.java | 2 +-
.../replicator/nrt/TestNRTReplication.java | 2 +-
.../nrt/TestStressNRTReplication.java | 2 +-
.../analysis/BaseTokenStreamTestCase.java | 4 +-
.../apache/lucene/geo/BaseGeoPointTestCase.java | 1 -
.../org/apache/lucene/index/RandomCodec.java | 7 +-
.../java/org/apache/lucene/util/TestUtil.java | 2 +-
solr/CHANGES.txt | 4 +
.../solr/collection1/conf/schema_codec.xml | 5 +-
.../org/apache/solr/core/TestCodecSupport.java | 9 +-
.../function/TestMinMaxOnMultiValuedField.java | 2 +-
.../function/TestSortByMinMaxFunction.java | 2 +-
.../TestDocTermOrdsUninvertLimit.java | 1 -
.../uninverting/TestFieldCacheVsDocValues.java | 111 +-
58 files changed, 62 insertions(+), 2799 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c00e77/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index ec68882..f6477f3 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -41,6 +41,9 @@ API Changes
* LUCENE-8242: Deprecated method IndexSearcher#createNormalizedWeight() has
been removed (Alan Woodward)
+* LUCENE-8267: Memory codecs removed from the codebase (MemoryPostings,
+ MemoryDocValues). (Dawid Weiss)
+
Changes in Runtime Behavior
* LUCENE-7837: Indices that were created before the previous major version
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c00e77/lucene/MIGRATE.txt
----------------------------------------------------------------------
diff --git a/lucene/MIGRATE.txt b/lucene/MIGRATE.txt
index 2825bca..0f9fcf6 100644
--- a/lucene/MIGRATE.txt
+++ b/lucene/MIGRATE.txt
@@ -43,3 +43,7 @@ the same index options for that field.
Instead use IndexSearcher.createWeight(), rewriting the query first, and using
a boost of 1f.
+
+## Memory codecs removed (LUCENE-8267) ##
+
+Memory codecs have been removed from the codebase (MemoryPostings, MemoryDocValues).
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c00e77/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
deleted file mode 100644
index eb7d096..0000000
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
+++ /dev/null
@@ -1,539 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.codecs.memory;
-
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.DocValuesConsumer;
-import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.codecs.LegacyDocValuesIterables;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.store.ByteArrayDataOutput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRefBuilder;
-import org.apache.lucene.util.MathUtil;
-import org.apache.lucene.util.fst.Builder;
-import org.apache.lucene.util.fst.FST.INPUT_TYPE;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.PositiveIntOutputs;
-import org.apache.lucene.util.fst.Util;
-import org.apache.lucene.util.packed.BlockPackedWriter;
-import org.apache.lucene.util.packed.MonotonicBlockPackedWriter;
-import org.apache.lucene.util.packed.PackedInts.FormatAndBits;
-import org.apache.lucene.util.packed.PackedInts;
-
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.BLOCK_COMPRESSED;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.BLOCK_SIZE;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.BYTES;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.DELTA_COMPRESSED;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.FST;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.GCD_COMPRESSED;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.NUMBER;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.SORTED_NUMERIC;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.SORTED_NUMERIC_SINGLETON;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.SORTED_SET;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.SORTED_SET_SINGLETON;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.TABLE_COMPRESSED;
-import static org.apache.lucene.codecs.memory.MemoryDocValuesProducer.VERSION_CURRENT;
-
-/**
- * Writer for {@link MemoryDocValuesFormat}
- */
-class MemoryDocValuesConsumer extends DocValuesConsumer {
- IndexOutput data, meta;
- final int maxDoc;
- final float acceptableOverheadRatio;
-
- MemoryDocValuesConsumer(SegmentWriteState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension, float acceptableOverheadRatio) throws IOException {
- this.acceptableOverheadRatio = acceptableOverheadRatio;
- maxDoc = state.segmentInfo.maxDoc();
- boolean success = false;
- try {
- String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
- data = state.directory.createOutput(dataName, state.context);
- CodecUtil.writeIndexHeader(data, dataCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
- String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
- meta = state.directory.createOutput(metaName, state.context);
- CodecUtil.writeIndexHeader(meta, metaCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(this);
- }
- }
- }
-
- @Override
- public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
- addNumericField(field, LegacyDocValuesIterables.numericIterable(field, valuesProducer, maxDoc), true);
- }
-
- void addNumericField(FieldInfo field, Iterable<Number> values, boolean optimizeStorage) throws IOException {
- meta.writeVInt(field.number);
- meta.writeByte(NUMBER);
- meta.writeLong(data.getFilePointer());
- long minValue = Long.MAX_VALUE;
- long maxValue = Long.MIN_VALUE;
- long blockSum = 0;
- long gcd = 0;
- boolean missing = false;
- // TODO: more efficient?
- HashSet<Long> uniqueValues = null;
- long count = 0;
-
- if (optimizeStorage) {
- uniqueValues = new HashSet<>();
-
- long currentBlockMin = Long.MAX_VALUE;
- long currentBlockMax = Long.MIN_VALUE;
- for (Number nv : values) {
- final long v;
- if (nv == null) {
- v = 0;
- missing = true;
- } else {
- v = nv.longValue();
- }
-
- if (gcd != 1) {
- if (v < Long.MIN_VALUE / 2 || v > Long.MAX_VALUE / 2) {
- // in that case v - minValue might overflow and make the GCD computation return
- // wrong results. Since these extreme values are unlikely, we just discard
- // GCD computation for them
- gcd = 1;
- } else if (count != 0) { // minValue needs to be set first
- gcd = MathUtil.gcd(gcd, v - minValue);
- }
- }
-
- currentBlockMin = Math.min(minValue, v);
- currentBlockMax = Math.max(maxValue, v);
-
- minValue = Math.min(minValue, v);
- maxValue = Math.max(maxValue, v);
-
- if (uniqueValues != null) {
- if (uniqueValues.add(v)) {
- if (uniqueValues.size() > 256) {
- uniqueValues = null;
- }
- }
- }
-
- ++count;
- if (count % BLOCK_SIZE == 0) {
- final long blockDelta = currentBlockMax - currentBlockMin;
- final int blockDeltaRequired = PackedInts.unsignedBitsRequired(blockDelta);
- final int blockBPV = PackedInts.fastestFormatAndBits(BLOCK_SIZE, blockDeltaRequired, acceptableOverheadRatio).bitsPerValue;
- blockSum += blockBPV;
- currentBlockMax = Long.MIN_VALUE;
- currentBlockMin = Long.MAX_VALUE;
- }
- }
- } else {
- for (Number nv : values) {
- long v = nv.longValue();
- maxValue = Math.max(v, maxValue);
- minValue = Math.min(v, minValue);
- count++;
- }
- }
-
- if (missing) {
- long start = data.getFilePointer();
- writeMissingBitset(values);
- meta.writeLong(start);
- meta.writeLong(data.getFilePointer() - start);
- } else {
- meta.writeLong(-1L);
- }
-
- final long delta = maxValue - minValue;
- final int deltaRequired = delta < 0 ? 64 : PackedInts.bitsRequired(delta);
- final FormatAndBits deltaBPV = PackedInts.fastestFormatAndBits(maxDoc, deltaRequired, acceptableOverheadRatio);
-
- final FormatAndBits tableBPV;
- if (count < Integer.MAX_VALUE && uniqueValues != null) {
- tableBPV = PackedInts.fastestFormatAndBits(maxDoc, PackedInts.bitsRequired(uniqueValues.size()-1), acceptableOverheadRatio);
- } else {
- tableBPV = null;
- }
-
- final FormatAndBits gcdBPV;
- if (count < Integer.MAX_VALUE && gcd != 0 && gcd != 1) {
- final long gcdDelta = (maxValue - minValue) / gcd;
- final int gcdRequired = gcdDelta < 0 ? 64 : PackedInts.bitsRequired(gcdDelta);
- gcdBPV = PackedInts.fastestFormatAndBits(maxDoc, gcdRequired, acceptableOverheadRatio);
- } else {
- gcdBPV = null;
- }
-
- boolean doBlock = false;
- if (blockSum != 0) {
- int numBlocks = maxDoc / BLOCK_SIZE;
- float avgBPV = blockSum / (float)numBlocks;
- // just a heuristic, with tiny amounts of blocks our estimate is skewed as we ignore the final "incomplete" block.
- // with at least 4 blocks it's pretty accurate. The difference must also be significant (according to acceptable overhead).
- if (numBlocks >= 4 && (avgBPV+avgBPV*acceptableOverheadRatio) < deltaBPV.bitsPerValue) {
- doBlock = true;
- }
- }
- // blockpackedreader allows us to read in huge streams of ints
- if (count >= Integer.MAX_VALUE) {
- doBlock = true;
- }
-
- if (tableBPV != null && (tableBPV.bitsPerValue+tableBPV.bitsPerValue*acceptableOverheadRatio) < deltaBPV.bitsPerValue) {
- // small number of unique values
- meta.writeByte(TABLE_COMPRESSED); // table-compressed
- Long[] decode = uniqueValues.toArray(new Long[uniqueValues.size()]);
- final HashMap<Long,Integer> encode = new HashMap<>();
- int length = 1 << tableBPV.bitsPerValue;
- data.writeVInt(length);
- for (int i = 0; i < decode.length; i++) {
- data.writeLong(decode[i]);
- encode.put(decode[i], i);
- }
- for (int i = decode.length; i < length; i++) {
- data.writeLong(0);
- }
-
- meta.writeVInt(PackedInts.VERSION_CURRENT);
- meta.writeLong(count);
- data.writeVInt(tableBPV.format.getId());
- data.writeVInt(tableBPV.bitsPerValue);
-
- final PackedInts.Writer writer = PackedInts.getWriterNoHeader(data, tableBPV.format, (int)count, tableBPV.bitsPerValue, PackedInts.DEFAULT_BUFFER_SIZE);
- for(Number nv : values) {
- writer.add(encode.get(nv == null ? 0 : nv.longValue()));
- }
- writer.finish();
- } else if (gcdBPV != null && (gcdBPV.bitsPerValue+gcdBPV.bitsPerValue*acceptableOverheadRatio) < deltaBPV.bitsPerValue) {
- meta.writeByte(GCD_COMPRESSED);
- meta.writeVInt(PackedInts.VERSION_CURRENT);
- meta.writeLong(count);
- data.writeLong(minValue);
- data.writeLong(gcd);
- data.writeVInt(gcdBPV.format.getId());
- data.writeVInt(gcdBPV.bitsPerValue);
-
- final PackedInts.Writer writer = PackedInts.getWriterNoHeader(data, gcdBPV.format, (int)count, gcdBPV.bitsPerValue, PackedInts.DEFAULT_BUFFER_SIZE);
- for (Number nv : values) {
- long value = nv == null ? 0 : nv.longValue();
- writer.add((value - minValue) / gcd);
- }
- writer.finish();
- } else if (doBlock) {
- meta.writeByte(BLOCK_COMPRESSED); // block delta-compressed
- meta.writeVInt(PackedInts.VERSION_CURRENT);
- meta.writeLong(count);
- data.writeVInt(BLOCK_SIZE);
- final BlockPackedWriter writer = new BlockPackedWriter(data, BLOCK_SIZE);
- for (Number nv : values) {
- writer.add(nv == null ? 0 : nv.longValue());
- }
- writer.finish();
- } else {
- meta.writeByte(DELTA_COMPRESSED); // delta-compressed
- meta.writeVInt(PackedInts.VERSION_CURRENT);
- meta.writeLong(count);
- final long minDelta = deltaBPV.bitsPerValue == 64 ? 0 : minValue;
- data.writeLong(minDelta);
- data.writeVInt(deltaBPV.format.getId());
- data.writeVInt(deltaBPV.bitsPerValue);
-
- final PackedInts.Writer writer = PackedInts.getWriterNoHeader(data, deltaBPV.format, (int)count, deltaBPV.bitsPerValue, PackedInts.DEFAULT_BUFFER_SIZE);
- for (Number nv : values) {
- long v = nv == null ? 0 : nv.longValue();
- writer.add(v - minDelta);
- }
- writer.finish();
- }
- }
-
- @Override
- public void close() throws IOException {
- boolean success = false;
- try {
- if (meta != null) {
- meta.writeVInt(-1); // write EOF marker
- CodecUtil.writeFooter(meta); // write checksum
- }
- if (data != null) {
- CodecUtil.writeFooter(data);
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(data, meta);
- } else {
- IOUtils.closeWhileHandlingException(data, meta);
- }
- data = meta = null;
- }
- }
-
- @Override
- public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
- addBinaryField(field, LegacyDocValuesIterables.binaryIterable(field, valuesProducer, maxDoc));
- }
-
- private void addBinaryField(FieldInfo field, final Iterable<BytesRef> values) throws IOException {
- // write the byte[] data
- meta.writeVInt(field.number);
- meta.writeByte(BYTES);
- int minLength = Integer.MAX_VALUE;
- int maxLength = Integer.MIN_VALUE;
- final long startFP = data.getFilePointer();
- boolean missing = false;
- int upto = 0;
- for(BytesRef v : values) {
- final int length;
- if (v == null) {
- length = 0;
- missing = true;
- } else {
- length = v.length;
- }
- if (length > MemoryDocValuesFormat.MAX_BINARY_FIELD_LENGTH) {
- throw new IllegalArgumentException("DocValuesField \"" + field.name + "\" is too large, must be <= " + MemoryDocValuesFormat.MAX_BINARY_FIELD_LENGTH + " but got length=" + length + " v=" + v + "; upto=" + upto + " values=" + values);
- }
- upto++;
- minLength = Math.min(minLength, length);
- maxLength = Math.max(maxLength, length);
- if (v != null) {
- data.writeBytes(v.bytes, v.offset, v.length);
- }
- }
- meta.writeLong(startFP);
- meta.writeLong(data.getFilePointer() - startFP);
- if (missing) {
- long start = data.getFilePointer();
- writeMissingBitset(values);
- meta.writeLong(start);
- meta.writeLong(data.getFilePointer() - start);
- } else {
- meta.writeLong(-1L);
- }
- meta.writeVInt(minLength);
- meta.writeVInt(maxLength);
-
- // if minLength == maxLength, it's a fixed-length byte[], we are done (the addresses are implicit)
- // otherwise, we need to record the length fields...
- if (minLength != maxLength) {
- meta.writeVInt(PackedInts.VERSION_CURRENT);
- meta.writeVInt(BLOCK_SIZE);
-
- final MonotonicBlockPackedWriter writer = new MonotonicBlockPackedWriter(data, BLOCK_SIZE);
- long addr = 0;
- for (BytesRef v : values) {
- if (v != null) {
- addr += v.length;
- }
- writer.add(addr);
- }
- writer.finish();
- }
- }
-
- private void writeFST(FieldInfo field, Iterable<BytesRef> values) throws IOException {
- meta.writeVInt(field.number);
- meta.writeByte(FST);
- meta.writeLong(data.getFilePointer());
- PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
- Builder<Long> builder = new Builder<>(INPUT_TYPE.BYTE1, outputs);
- IntsRefBuilder scratch = new IntsRefBuilder();
- long ord = 0;
- for (BytesRef v : values) {
- builder.add(Util.toIntsRef(v, scratch), ord);
- ord++;
- }
- FST<Long> fst = builder.finish();
- if (fst != null) {
- fst.save(data);
- }
- meta.writeVLong(ord);
- }
-
- // TODO: in some cases representing missing with minValue-1 wouldn't take up additional space and so on,
- // but this is very simple, and algorithms only check this for values of 0 anyway (doesnt slow down normal decode)
- void writeMissingBitset(Iterable<?> values) throws IOException {
- long bits = 0;
- int count = 0;
- for (Object v : values) {
- if (count == 64) {
- data.writeLong(bits);
- count = 0;
- bits = 0;
- }
- if (v != null) {
- bits |= 1L << (count & 0x3f);
- }
- count++;
- }
- if (count > 0) {
- data.writeLong(bits);
- }
- }
-
- @Override
- public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
- addSortedField(field,
- LegacyDocValuesIterables.valuesIterable(valuesProducer.getSorted(field)),
- LegacyDocValuesIterables.sortedOrdIterable(valuesProducer, field, maxDoc));
- }
-
- private void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
- // write the ordinals as numerics
- addNumericField(field, docToOrd, false);
-
- // write the values as FST
- writeFST(field, values);
- }
-
- @Override
- public void addSortedNumericField(FieldInfo field, final DocValuesProducer valuesProducer) throws IOException {
-
- final Iterable<Number> docToValueCount = LegacyDocValuesIterables.sortedNumericToDocCount(valuesProducer, field, maxDoc);
- final Iterable<Number> values = LegacyDocValuesIterables.sortedNumericToValues(valuesProducer, field);
-
- meta.writeVInt(field.number);
-
- if (isSingleValued(docToValueCount)) {
- meta.writeByte(SORTED_NUMERIC_SINGLETON);
- addNumericField(field, singletonView(docToValueCount, values, null), true);
- } else {
- meta.writeByte(SORTED_NUMERIC);
-
- // write the addresses:
- meta.writeVInt(PackedInts.VERSION_CURRENT);
- meta.writeVInt(BLOCK_SIZE);
- meta.writeLong(data.getFilePointer());
- final MonotonicBlockPackedWriter writer = new MonotonicBlockPackedWriter(data, BLOCK_SIZE);
- long addr = 0;
- writer.add(addr);
- for (Number v : docToValueCount) {
- addr += v.longValue();
- writer.add(addr);
- }
- writer.finish();
- long valueCount = writer.ord();
- meta.writeLong(valueCount);
-
- // write the values
- addNumericField(field, values, true);
- }
- }
-
- // note: this might not be the most efficient... but it's fairly simple
- @Override
- public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
- Iterable<BytesRef> values = LegacyDocValuesIterables.valuesIterable(valuesProducer.getSortedSet(field));
- Iterable<Number> docToOrdCount = LegacyDocValuesIterables.sortedSetOrdCountIterable(valuesProducer, field, maxDoc);
- Iterable<Number> ords = LegacyDocValuesIterables.sortedSetOrdsIterable(valuesProducer, field);
- meta.writeVInt(field.number);
-
- if (isSingleValued(docToOrdCount)) {
- meta.writeByte(SORTED_SET_SINGLETON);
- addSortedField(field, values, singletonView(docToOrdCount, ords, -1L));
- } else {
- meta.writeByte(SORTED_SET);
- // write the ordinals as a binary field
- addBinaryField(field, new Iterable<BytesRef>() {
- @Override
- public Iterator<BytesRef> iterator() {
- return new SortedSetIterator(docToOrdCount.iterator(), ords.iterator());
- }
- });
-
- // write the values as FST
- writeFST(field, values);
- }
- }
-
- // per-document vint-encoded byte[]
- static class SortedSetIterator implements Iterator<BytesRef> {
- byte[] buffer = new byte[10];
- ByteArrayDataOutput out = new ByteArrayDataOutput();
- BytesRef ref = new BytesRef();
-
- final Iterator<Number> counts;
- final Iterator<Number> ords;
-
- SortedSetIterator(Iterator<Number> counts, Iterator<Number> ords) {
- this.counts = counts;
- this.ords = ords;
- }
-
- @Override
- public boolean hasNext() {
- return counts.hasNext();
- }
-
- @Override
- public BytesRef next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
-
- int count = counts.next().intValue();
- int maxSize = count*9; // worst case
- if (maxSize > buffer.length) {
- buffer = ArrayUtil.grow(buffer, maxSize);
- }
-
- try {
- encodeValues(count);
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
-
- ref.bytes = buffer;
- ref.offset = 0;
- ref.length = out.getPosition();
-
- return ref;
- }
-
- // encodes count values to buffer
- private void encodeValues(int count) throws IOException {
- out.reset(buffer);
- long lastOrd = 0;
- for (int i = 0; i < count; i++) {
- long ord = ords.next().longValue();
- out.writeVLong(ord - lastOrd);
- lastOrd = ord;
- }
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c00e77/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
deleted file mode 100644
index 4624ec0..0000000
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.codecs.memory;
-
-
-import java.io.IOException;
-
-import org.apache.lucene.codecs.DocValuesConsumer;
-import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.codecs.DocValuesFormat;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.packed.PackedInts;
-
-/** In-memory docvalues format */
-public class MemoryDocValuesFormat extends DocValuesFormat {
-
- /** Maximum length for each binary doc values field. */
- public static final int MAX_BINARY_FIELD_LENGTH = (1 << 15) - 2;
-
- final float acceptableOverheadRatio;
-
- /**
- * Calls {@link #MemoryDocValuesFormat(float)
- * MemoryDocValuesFormat(PackedInts.DEFAULT)}
- */
- public MemoryDocValuesFormat() {
- this(PackedInts.DEFAULT);
- }
-
- /**
- * Creates a new MemoryDocValuesFormat with the specified
- * <code>acceptableOverheadRatio</code> for NumericDocValues.
- * @param acceptableOverheadRatio compression parameter for numerics.
- * Currently this is only used when the number of unique values is small.
- *
- * @lucene.experimental
- */
- public MemoryDocValuesFormat(float acceptableOverheadRatio) {
- super("Memory");
- this.acceptableOverheadRatio = acceptableOverheadRatio;
- }
-
- @Override
- public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- return new MemoryDocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION, acceptableOverheadRatio);
- }
-
- @Override
- public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
- return new MemoryDocValuesProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
- }
-
- static final String DATA_CODEC = "MemoryDocValuesData";
- static final String DATA_EXTENSION = "mdvd";
- static final String METADATA_CODEC = "MemoryDocValuesMetadata";
- static final String METADATA_EXTENSION = "mdvm";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c00e77/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
deleted file mode 100644
index ac81360..0000000
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
+++ /dev/null
@@ -1,877 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.codecs.memory;
-
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.index.*;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.Accountables;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRef;
-import org.apache.lucene.util.IntsRefBuilder;
-import org.apache.lucene.util.PagedBytes;
-import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.lucene.util.fst.BytesRefFSTEnum;
-import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.FST.Arc;
-import org.apache.lucene.util.fst.FST.BytesReader;
-import org.apache.lucene.util.fst.PositiveIntOutputs;
-import org.apache.lucene.util.fst.Util;
-import org.apache.lucene.util.packed.BlockPackedReader;
-import org.apache.lucene.util.packed.MonotonicBlockPackedReader;
-import org.apache.lucene.util.packed.PackedInts;
-
-/**
- * Reader for {@link MemoryDocValuesFormat}
- */
-class MemoryDocValuesProducer extends DocValuesProducer {
- // metadata maps (just file pointers and minimal stuff)
- private final Map<String,NumericEntry> numerics = new HashMap<>();
- private final Map<String,BinaryEntry> binaries = new HashMap<>();
- private final Map<String,FSTEntry> fsts = new HashMap<>();
- private final Map<String,SortedSetEntry> sortedSets = new HashMap<>();
- private final Map<String,SortedNumericEntry> sortedNumerics = new HashMap<>();
- private final IndexInput data;
-
- // ram instances we have already loaded
- private final Map<String,LegacyNumericDocValues> numericInstances = new HashMap<>();
- private final Map<String,BytesAndAddresses> pagedBytesInstances = new HashMap<>();
- private final Map<String,FST<Long>> fstInstances = new HashMap<>();
- private final Map<String,FixedBitSet> docsWithFieldInstances = new HashMap<>();
- private final Map<String,MonotonicBlockPackedReader> addresses = new HashMap<>();
-
- private final Map<String,Accountable> numericInfo = new HashMap<>();
-
- private final int numEntries;
- private final int maxDoc;
- private final AtomicLong ramBytesUsed;
- private final int version;
-
- private final boolean merging;
-
- static final byte NUMBER = 0;
- static final byte BYTES = 1;
- static final byte FST = 2;
- static final byte SORTED_SET = 4;
- static final byte SORTED_SET_SINGLETON = 5;
- static final byte SORTED_NUMERIC = 6;
- static final byte SORTED_NUMERIC_SINGLETON = 7;
-
- static final int BLOCK_SIZE = 4096;
-
- static final byte DELTA_COMPRESSED = 0;
- static final byte TABLE_COMPRESSED = 1;
- static final byte BLOCK_COMPRESSED = 2;
- static final byte GCD_COMPRESSED = 3;
-
- static final int VERSION_START = 4;
- static final int VERSION_CURRENT = VERSION_START;
-
- // clone for merge: when merging we don't do any instances.put()s
- MemoryDocValuesProducer(MemoryDocValuesProducer original) throws IOException {
- assert Thread.holdsLock(original);
- numerics.putAll(original.numerics);
- binaries.putAll(original.binaries);
- fsts.putAll(original.fsts);
- sortedSets.putAll(original.sortedSets);
- sortedNumerics.putAll(original.sortedNumerics);
- data = original.data.clone();
-
- numericInstances.putAll(original.numericInstances);
- pagedBytesInstances.putAll(original.pagedBytesInstances);
- fstInstances.putAll(original.fstInstances);
- docsWithFieldInstances.putAll(original.docsWithFieldInstances);
- addresses.putAll(original.addresses);
-
- numericInfo.putAll(original.numericInfo);
-
- numEntries = original.numEntries;
- maxDoc = original.maxDoc;
- ramBytesUsed = new AtomicLong(original.ramBytesUsed.get());
- version = original.version;
- merging = true;
- }
-
- MemoryDocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
- maxDoc = state.segmentInfo.maxDoc();
- merging = false;
- String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
- // read in the entries from the metadata file.
- ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
- boolean success = false;
- try {
- version = CodecUtil.checkIndexHeader(in, metaCodec, VERSION_START, VERSION_CURRENT,
- state.segmentInfo.getId(), state.segmentSuffix);
- numEntries = readFields(in, state.fieldInfos);
- CodecUtil.checkFooter(in);
- ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
- success = true;
- } finally {
- if (success) {
- IOUtils.close(in);
- } else {
- IOUtils.closeWhileHandlingException(in);
- }
- }
-
- String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
- this.data = state.directory.openInput(dataName, state.context);
- success = false;
- try {
- final int version2 = CodecUtil.checkIndexHeader(data, dataCodec, VERSION_START, VERSION_CURRENT,
- state.segmentInfo.getId(), state.segmentSuffix);
- if (version != version2) {
- throw new CorruptIndexException("Format versions mismatch: meta=" + version + ", data=" + version2, data);
- }
-
- // NOTE: data file is too costly to verify checksum against all the bytes on open,
- // but for now we at least verify proper structure of the checksum footer: which looks
- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
- // such as file truncation.
- CodecUtil.retrieveChecksum(data);
-
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(this.data);
- }
- }
- }
-
- private NumericEntry readNumericEntry(IndexInput meta) throws IOException {
- NumericEntry entry = new NumericEntry();
- entry.offset = meta.readLong();
- entry.missingOffset = meta.readLong();
- if (entry.missingOffset != -1) {
- entry.missingBytes = meta.readLong();
- } else {
- entry.missingBytes = 0;
- }
- entry.format = meta.readByte();
- switch(entry.format) {
- case DELTA_COMPRESSED:
- case TABLE_COMPRESSED:
- case BLOCK_COMPRESSED:
- case GCD_COMPRESSED:
- break;
- default:
- throw new CorruptIndexException("Unknown format: " + entry.format, meta);
- }
- entry.packedIntsVersion = meta.readVInt();
- entry.count = meta.readLong();
- return entry;
- }
-
- private BinaryEntry readBinaryEntry(IndexInput meta) throws IOException {
- BinaryEntry entry = new BinaryEntry();
- entry.offset = meta.readLong();
- entry.numBytes = meta.readLong();
- entry.missingOffset = meta.readLong();
- if (entry.missingOffset != -1) {
- entry.missingBytes = meta.readLong();
- } else {
- entry.missingBytes = 0;
- }
- entry.minLength = meta.readVInt();
- entry.maxLength = meta.readVInt();
- if (entry.minLength != entry.maxLength) {
- entry.packedIntsVersion = meta.readVInt();
- entry.blockSize = meta.readVInt();
- }
- return entry;
- }
-
- private FSTEntry readFSTEntry(IndexInput meta) throws IOException {
- FSTEntry entry = new FSTEntry();
- entry.offset = meta.readLong();
- entry.numOrds = meta.readVLong();
- return entry;
- }
-
- private int readFields(IndexInput meta, FieldInfos infos) throws IOException {
- int numEntries = 0;
- int fieldNumber = meta.readVInt();
- while (fieldNumber != -1) {
- numEntries++;
- FieldInfo info = infos.fieldInfo(fieldNumber);
- if (info == null) {
- throw new CorruptIndexException("invalid field number: " + fieldNumber, meta);
- }
- int fieldType = meta.readByte();
- if (fieldType == NUMBER) {
- numerics.put(info.name, readNumericEntry(meta));
- } else if (fieldType == BYTES) {
- binaries.put(info.name, readBinaryEntry(meta));
- } else if (fieldType == FST) {
- fsts.put(info.name,readFSTEntry(meta));
- } else if (fieldType == SORTED_SET) {
- SortedSetEntry entry = new SortedSetEntry();
- entry.singleton = false;
- sortedSets.put(info.name, entry);
- } else if (fieldType == SORTED_SET_SINGLETON) {
- SortedSetEntry entry = new SortedSetEntry();
- entry.singleton = true;
- sortedSets.put(info.name, entry);
- } else if (fieldType == SORTED_NUMERIC) {
- SortedNumericEntry entry = new SortedNumericEntry();
- entry.singleton = false;
- entry.packedIntsVersion = meta.readVInt();
- entry.blockSize = meta.readVInt();
- entry.addressOffset = meta.readLong();
- entry.valueCount = meta.readLong();
- sortedNumerics.put(info.name, entry);
- } else if (fieldType == SORTED_NUMERIC_SINGLETON) {
- SortedNumericEntry entry = new SortedNumericEntry();
- entry.singleton = true;
- sortedNumerics.put(info.name, entry);
- } else {
- throw new CorruptIndexException("invalid entry type: " + fieldType + ", fieldName=" + info.name, meta);
- }
- fieldNumber = meta.readVInt();
- }
- return numEntries;
- }
-
- @Override
- public synchronized NumericDocValues getNumeric(FieldInfo field) throws IOException {
- LegacyNumericDocValues instance = numericInstances.get(field.name);
- if (instance == null) {
- instance = loadNumeric(field);
- if (!merging) {
- numericInstances.put(field.name, instance);
- }
- }
- NumericEntry ne = numerics.get(field.name);
- return new LegacyNumericDocValuesWrapper(getMissingBits(field, ne.missingOffset, ne.missingBytes), instance);
- }
-
- private synchronized LegacyNumericDocValues getNumericNonIterator(FieldInfo field) throws IOException {
- LegacyNumericDocValues instance = numericInstances.get(field.name);
- if (instance == null) {
- instance = loadNumeric(field);
- if (!merging) {
- numericInstances.put(field.name, instance);
- }
- }
- return instance;
- }
-
- @Override
- public long ramBytesUsed() {
- return ramBytesUsed.get();
- }
-
- @Override
- public synchronized Collection<Accountable> getChildResources() {
- List<Accountable> resources = new ArrayList<>();
- resources.addAll(Accountables.namedAccountables("numeric field", numericInfo));
- resources.addAll(Accountables.namedAccountables("pagedbytes field", pagedBytesInstances));
- resources.addAll(Accountables.namedAccountables("term dict field", fstInstances));
- resources.addAll(Accountables.namedAccountables("missing bitset field", docsWithFieldInstances));
- resources.addAll(Accountables.namedAccountables("addresses field", addresses));
- return Collections.unmodifiableList(resources);
- }
-
- @Override
- public void checkIntegrity() throws IOException {
- CodecUtil.checksumEntireFile(data.clone());
- }
-
- @Override
- public synchronized DocValuesProducer getMergeInstance() throws IOException {
- return new MemoryDocValuesProducer(this);
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName() + "(entries=" + numEntries + ")";
- }
-
- private LegacyNumericDocValues loadNumeric(FieldInfo field) throws IOException {
- NumericEntry entry = numerics.get(field.name);
- IndexInput data = this.data.clone();
- data.seek(entry.offset + entry.missingBytes);
- switch (entry.format) {
- case TABLE_COMPRESSED:
- int size = data.readVInt();
- if (size > 256) {
- throw new CorruptIndexException("TABLE_COMPRESSED cannot have more than 256 distinct values, got=" + size, data);
- }
- final long decode[] = new long[size];
- for (int i = 0; i < decode.length; i++) {
- decode[i] = data.readLong();
- }
- final int formatID = data.readVInt();
- final int bitsPerValue = data.readVInt();
- final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, (int)entry.count, bitsPerValue);
- if (!merging) {
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed());
- numericInfo.put(field.name, Accountables.namedAccountable("table compressed", ordsReader));
- }
- return new LegacyNumericDocValues() {
- @Override
- public long get(int docID) {
- return decode[(int)ordsReader.get(docID)];
- }
- };
- case DELTA_COMPRESSED:
- final long minDelta = data.readLong();
- final int formatIDDelta = data.readVInt();
- final int bitsPerValueDelta = data.readVInt();
- final PackedInts.Reader deltaReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatIDDelta), entry.packedIntsVersion, (int)entry.count, bitsPerValueDelta);
- if (!merging) {
- ramBytesUsed.addAndGet(deltaReader.ramBytesUsed());
- numericInfo.put(field.name, Accountables.namedAccountable("delta compressed", deltaReader));
- }
- return new LegacyNumericDocValues() {
- @Override
- public long get(int docID) {
- return minDelta + deltaReader.get(docID);
- }
- };
- case BLOCK_COMPRESSED:
- final int blockSize = data.readVInt();
- final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, entry.count, false);
- if (!merging) {
- ramBytesUsed.addAndGet(reader.ramBytesUsed());
- numericInfo.put(field.name, Accountables.namedAccountable("block compressed", reader));
- }
- return new LegacyNumericDocValues() {
- @Override
- public long get(int docID) {
- return reader.get(docID);
- }
- };
- case GCD_COMPRESSED:
- final long min = data.readLong();
- final long mult = data.readLong();
- final int formatIDGCD = data.readVInt();
- final int bitsPerValueGCD = data.readVInt();
- final PackedInts.Reader quotientReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatIDGCD), entry.packedIntsVersion, (int)entry.count, bitsPerValueGCD);
- if (!merging) {
- ramBytesUsed.addAndGet(quotientReader.ramBytesUsed());
- numericInfo.put(field.name, Accountables.namedAccountable("gcd compressed", quotientReader));
- }
- return new LegacyNumericDocValues() {
- @Override
- public long get(int docID) {
- return min + mult * quotientReader.get(docID);
- }
- };
- default:
- throw new AssertionError();
- }
- }
-
- private LegacyBinaryDocValues getLegacyBinary(FieldInfo field) throws IOException {
- BinaryEntry entry = binaries.get(field.name);
-
- BytesAndAddresses instance;
- synchronized (this) {
- instance = pagedBytesInstances.get(field.name);
- if (instance == null) {
- instance = loadBinary(field);
- if (!merging) {
- pagedBytesInstances.put(field.name, instance);
- }
- }
- }
- final PagedBytes.Reader bytesReader = instance.reader;
- final MonotonicBlockPackedReader addresses = instance.addresses;
-
- if (addresses == null) {
- assert entry.minLength == entry.maxLength;
- final int fixedLength = entry.minLength;
- return new LegacyBinaryDocValues() {
- final BytesRef term = new BytesRef();
-
- @Override
- public BytesRef get(int docID) {
- bytesReader.fillSlice(term, fixedLength * (long)docID, fixedLength);
- return term;
- }
- };
- } else {
- return new LegacyBinaryDocValues() {
- final BytesRef term = new BytesRef();
-
- @Override
- public BytesRef get(int docID) {
- long startAddress = docID == 0 ? 0 : addresses.get(docID-1);
- long endAddress = addresses.get(docID);
- bytesReader.fillSlice(term, startAddress, (int) (endAddress - startAddress));
- return term;
- }
- };
- }
- }
-
- @Override
- public synchronized BinaryDocValues getBinary(FieldInfo field) throws IOException {
- BinaryEntry be = binaries.get(field.name);
- return new LegacyBinaryDocValuesWrapper(getMissingBits(field, be.missingOffset, be.missingBytes), getLegacyBinary(field));
- }
-
- private BytesAndAddresses loadBinary(FieldInfo field) throws IOException {
- BytesAndAddresses bytesAndAddresses = new BytesAndAddresses();
- BinaryEntry entry = binaries.get(field.name);
- IndexInput data = this.data.clone();
- data.seek(entry.offset);
- PagedBytes bytes = new PagedBytes(16);
- bytes.copy(data, entry.numBytes);
- bytesAndAddresses.reader = bytes.freeze(true);
- if (!merging) {
- ramBytesUsed.addAndGet(bytesAndAddresses.reader.ramBytesUsed());
- }
- if (entry.minLength != entry.maxLength) {
- data.seek(data.getFilePointer() + entry.missingBytes);
- bytesAndAddresses.addresses = MonotonicBlockPackedReader.of(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
- if (!merging) {
- ramBytesUsed.addAndGet(bytesAndAddresses.addresses.ramBytesUsed());
- }
- }
- return bytesAndAddresses;
- }
-
- @Override
- public SortedDocValues getSorted(FieldInfo field) throws IOException {
- return new LegacySortedDocValuesWrapper(getSortedNonIterator(field), maxDoc);
- }
-
- private LegacySortedDocValues getSortedNonIterator(FieldInfo field) throws IOException {
- final FSTEntry entry = fsts.get(field.name);
- if (entry.numOrds == 0) {
- return DocValues.emptyLegacySorted();
- }
- FST<Long> instance;
- synchronized(this) {
- instance = fstInstances.get(field.name);
- if (instance == null) {
- IndexInput data = this.data.clone();
- data.seek(entry.offset);
- instance = new FST<>(data, PositiveIntOutputs.getSingleton());
- if (!merging) {
- ramBytesUsed.addAndGet(instance.ramBytesUsed());
- fstInstances.put(field.name, instance);
- }
- }
- }
- final LegacyNumericDocValues docToOrd = getNumericNonIterator(field);
- final FST<Long> fst = instance;
-
- // per-thread resources
- final BytesReader in = fst.getBytesReader();
- final Arc<Long> firstArc = new Arc<>();
- final Arc<Long> scratchArc = new Arc<>();
- final IntsRefBuilder scratchInts = new IntsRefBuilder();
- final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<>(fst);
-
- return new LegacySortedDocValues() {
- final BytesRefBuilder term = new BytesRefBuilder();
-
- @Override
- public int getOrd(int docID) {
- return (int) docToOrd.get(docID);
- }
-
- @Override
- public BytesRef lookupOrd(int ord) {
- try {
- in.setPosition(0);
- fst.getFirstArc(firstArc);
- IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts);
- return Util.toBytesRef(output, term);
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
- }
-
- @Override
- public int lookupTerm(BytesRef key) {
- try {
- InputOutput<Long> o = fstEnum.seekCeil(key);
- if (o == null) {
- return -getValueCount()-1;
- } else if (o.input.equals(key)) {
- return o.output.intValue();
- } else {
- return (int) -o.output-1;
- }
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
- }
-
- @Override
- public int getValueCount() {
- return (int)entry.numOrds;
- }
-
- @Override
- public TermsEnum termsEnum() {
- return new FSTTermsEnum(fst);
- }
- };
- }
-
- @Override
- public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
- SortedNumericEntry entry = sortedNumerics.get(field.name);
- if (entry.singleton) {
- LegacyNumericDocValues values = getNumericNonIterator(field);
- NumericEntry ne = numerics.get(field.name);
- Bits docsWithField = getMissingBits(field, ne.missingOffset, ne.missingBytes);
- return DocValues.singleton(new LegacyNumericDocValuesWrapper(docsWithField, values));
- } else {
- final LegacyNumericDocValues values = getNumericNonIterator(field);
- final MonotonicBlockPackedReader addr;
- synchronized (this) {
- MonotonicBlockPackedReader res = addresses.get(field.name);
- if (res == null) {
- IndexInput data = this.data.clone();
- data.seek(entry.addressOffset);
- res = MonotonicBlockPackedReader.of(data, entry.packedIntsVersion, entry.blockSize, entry.valueCount, false);
- if (!merging) {
- addresses.put(field.name, res);
- ramBytesUsed.addAndGet(res.ramBytesUsed());
- }
- }
- addr = res;
- }
- return new LegacySortedNumericDocValuesWrapper(new LegacySortedNumericDocValues() {
- int startOffset;
- int endOffset;
-
- @Override
- public void setDocument(int doc) {
- startOffset = (int) addr.get(doc);
- endOffset = (int) addr.get(doc+1);
- }
-
- @Override
- public long valueAt(int index) {
- return values.get(startOffset + index);
- }
-
- @Override
- public int count() {
- return (endOffset - startOffset);
- }
- }, maxDoc);
- }
- }
-
- @Override
- public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
- SortedSetEntry sortedSetEntry = sortedSets.get(field.name);
- if (sortedSetEntry.singleton) {
- return DocValues.singleton(getSorted(field));
- }
-
- final FSTEntry entry = fsts.get(field.name);
- if (entry.numOrds == 0) {
- return DocValues.emptySortedSet(); // empty FST!
- }
- FST<Long> instance;
- synchronized(this) {
- instance = fstInstances.get(field.name);
- if (instance == null) {
- IndexInput data = this.data.clone();
- data.seek(entry.offset);
- instance = new FST<>(data, PositiveIntOutputs.getSingleton());
- if (!merging) {
- ramBytesUsed.addAndGet(instance.ramBytesUsed());
- fstInstances.put(field.name, instance);
- }
- }
- }
- final LegacyBinaryDocValues docToOrds = getLegacyBinary(field);
- final FST<Long> fst = instance;
-
- // per-thread resources
- final BytesReader in = fst.getBytesReader();
- final Arc<Long> firstArc = new Arc<>();
- final Arc<Long> scratchArc = new Arc<>();
- final IntsRefBuilder scratchInts = new IntsRefBuilder();
- final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<>(fst);
- final ByteArrayDataInput input = new ByteArrayDataInput();
- return new LegacySortedSetDocValuesWrapper(new LegacySortedSetDocValues() {
- final BytesRefBuilder term = new BytesRefBuilder();
- BytesRef ref;
- long currentOrd;
-
- @Override
- public long nextOrd() {
- if (input.eof()) {
- return NO_MORE_ORDS;
- } else {
- currentOrd += input.readVLong();
- return currentOrd;
- }
- }
-
- @Override
- public void setDocument(int docID) {
- ref = docToOrds.get(docID);
- input.reset(ref.bytes, ref.offset, ref.length);
- currentOrd = 0;
- }
-
- @Override
- public BytesRef lookupOrd(long ord) {
- try {
- in.setPosition(0);
- fst.getFirstArc(firstArc);
- IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts);
- return Util.toBytesRef(output, term);
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
- }
-
- @Override
- public long lookupTerm(BytesRef key) {
- try {
- InputOutput<Long> o = fstEnum.seekCeil(key);
- if (o == null) {
- return -getValueCount()-1;
- } else if (o.input.equals(key)) {
- return o.output.intValue();
- } else {
- return -o.output-1;
- }
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
- }
-
- @Override
- public long getValueCount() {
- return entry.numOrds;
- }
-
- @Override
- public TermsEnum termsEnum() {
- return new FSTTermsEnum(fst);
- }
- }, maxDoc);
- }
-
- private Bits getMissingBits(FieldInfo field, final long offset, final long length) throws IOException {
- if (offset == -1) {
- return new Bits.MatchAllBits(maxDoc);
- } else {
- FixedBitSet instance;
- synchronized(this) {
- instance = docsWithFieldInstances.get(field.name);
- if (instance == null) {
- IndexInput data = this.data.clone();
- data.seek(offset);
- assert length % 8 == 0;
- long bits[] = new long[(int) length >> 3];
- for (int i = 0; i < bits.length; i++) {
- bits[i] = data.readLong();
- }
- instance = new FixedBitSet(bits, maxDoc);
- if (!merging) {
- docsWithFieldInstances.put(field.name, instance);
- ramBytesUsed.addAndGet(instance.ramBytesUsed());
- }
- }
- }
- return instance;
- }
- }
-
- @Override
- public void close() throws IOException {
- data.close();
- }
-
- static class NumericEntry {
- long offset;
- long count;
- long missingOffset;
- long missingBytes;
- byte format;
- int packedIntsVersion;
- }
-
- static class BinaryEntry {
- long offset;
- long missingOffset;
- long missingBytes;
- long numBytes;
- int minLength;
- int maxLength;
- int packedIntsVersion;
- int blockSize;
- }
-
- static class FSTEntry {
- long offset;
- long numOrds;
- }
-
- static class SortedSetEntry {
- boolean singleton;
- }
-
- static class SortedNumericEntry {
- boolean singleton;
- long addressOffset;
- int packedIntsVersion;
- int blockSize;
- long valueCount;
- }
-
- static class BytesAndAddresses implements Accountable {
- PagedBytes.Reader reader;
- MonotonicBlockPackedReader addresses;
-
- @Override
- public long ramBytesUsed() {
- long bytesUsed = reader.ramBytesUsed();
- if (addresses != null) {
- bytesUsed += addresses.ramBytesUsed();
- }
- return bytesUsed;
- }
-
- @Override
- public Collection<Accountable> getChildResources() {
- List<Accountable> resources = new ArrayList<>();
- if (addresses != null) {
- resources.add(Accountables.namedAccountable("addresses", addresses));
- }
- resources.add(Accountables.namedAccountable("term bytes", reader));
- return Collections.unmodifiableList(resources);
- }
- }
-
- // exposes FSTEnum directly as a TermsEnum: avoids binary-search next()
- static class FSTTermsEnum extends TermsEnum {
- final BytesRefFSTEnum<Long> in;
-
- // this is all for the complicated seek(ord)...
- // maybe we should add a FSTEnum that supports this operation?
- final FST<Long> fst;
- final FST.BytesReader bytesReader;
- final Arc<Long> firstArc = new Arc<>();
- final Arc<Long> scratchArc = new Arc<>();
- final IntsRefBuilder scratchInts = new IntsRefBuilder();
- final BytesRefBuilder scratchBytes = new BytesRefBuilder();
-
- FSTTermsEnum(FST<Long> fst) {
- this.fst = fst;
- in = new BytesRefFSTEnum<>(fst);
- bytesReader = fst.getBytesReader();
- }
-
- @Override
- public BytesRef next() throws IOException {
- InputOutput<Long> io = in.next();
- if (io == null) {
- return null;
- } else {
- return io.input;
- }
- }
-
- @Override
- public SeekStatus seekCeil(BytesRef text) throws IOException {
- if (in.seekCeil(text) == null) {
- return SeekStatus.END;
- } else if (term().equals(text)) {
- // TODO: add SeekStatus to FSTEnum like in https://issues.apache.org/jira/browse/LUCENE-3729
- // to remove this comparision?
- return SeekStatus.FOUND;
- } else {
- return SeekStatus.NOT_FOUND;
- }
- }
-
- @Override
- public boolean seekExact(BytesRef text) throws IOException {
- if (in.seekExact(text) == null) {
- return false;
- } else {
- return true;
- }
- }
-
- @Override
- public void seekExact(long ord) throws IOException {
- // TODO: would be better to make this simpler and faster.
- // but we dont want to introduce a bug that corrupts our enum state!
- bytesReader.setPosition(0);
- fst.getFirstArc(firstArc);
- IntsRef output = Util.getByOutput(fst, ord, bytesReader, firstArc, scratchArc, scratchInts);
- // TODO: we could do this lazily, better to try to push into FSTEnum though?
- in.seekExact(Util.toBytesRef(output, new BytesRefBuilder()));
- }
-
- @Override
- public BytesRef term() throws IOException {
- return in.current().input;
- }
-
- @Override
- public long ord() throws IOException {
- return in.current().output;
- }
-
- @Override
- public int docFreq() throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long totalTermFreq() throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ImpactsEnum impacts(int flags) throws IOException {
- throw new UnsupportedOperationException();
- }
- }
-}