You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/09/16 15:26:58 UTC
svn commit: r1625275 [3/4] - in /lucene/dev/trunk: lucene/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lu...
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/Lucene410DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/Lucene410DocValuesProducer.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/Lucene410DocValuesProducer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/Lucene410DocValuesProducer.java Tue Sep 16 13:26:55 2014
@@ -36,7 +36,10 @@ import static org.apache.lucene.codecs.l
import java.io.Closeable; // javadocs
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
@@ -60,6 +63,8 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -71,21 +76,22 @@ import org.apache.lucene.util.packed.Mon
/** reader for {@link Lucene410DocValuesFormat} */
class Lucene410DocValuesProducer extends DocValuesProducer implements Closeable {
- private final Map<Integer,NumericEntry> numerics;
- private final Map<Integer,BinaryEntry> binaries;
- private final Map<Integer,SortedSetEntry> sortedSets;
- private final Map<Integer,SortedSetEntry> sortedNumerics;
- private final Map<Integer,NumericEntry> ords;
- private final Map<Integer,NumericEntry> ordIndexes;
+ private final Map<String,NumericEntry> numerics;
+ private final Map<String,BinaryEntry> binaries;
+ private final Map<String,SortedSetEntry> sortedSets;
+ private final Map<String,SortedSetEntry> sortedNumerics;
+ private final Map<String,NumericEntry> ords;
+ private final Map<String,NumericEntry> ordIndexes;
+ private final int numFields;
private final AtomicLong ramBytesUsed;
private final IndexInput data;
private final int maxDoc;
private final int version;
// memory-resident structures
- private final Map<Integer,MonotonicBlockPackedReader> addressInstances = new HashMap<>();
- private final Map<Integer,MonotonicBlockPackedReader> ordIndexInstances = new HashMap<>();
- private final Map<Integer,ReverseTermsIndex> reverseIndexInstances = new HashMap<>();
+ private final Map<String,MonotonicBlockPackedReader> addressInstances = new HashMap<>();
+ private final Map<String,MonotonicBlockPackedReader> ordIndexInstances = new HashMap<>();
+ private final Map<String,ReverseTermsIndex> reverseIndexInstances = new HashMap<>();
/** expert: instantiates a new reader */
Lucene410DocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
@@ -104,7 +110,7 @@ class Lucene410DocValuesProducer extends
binaries = new HashMap<>();
sortedSets = new HashMap<>();
sortedNumerics = new HashMap<>();
- readFields(in, state.fieldInfos);
+ numFields = readFields(in, state.fieldInfos);
CodecUtil.checkFooter(in);
success = true;
@@ -143,108 +149,110 @@ class Lucene410DocValuesProducer extends
ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
}
- private void readSortedField(int fieldNumber, IndexInput meta, FieldInfos infos) throws IOException {
+ private void readSortedField(FieldInfo info, IndexInput meta) throws IOException {
// sorted = binary + numeric
- if (meta.readVInt() != fieldNumber) {
- throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ if (meta.readVInt() != info.number) {
+ throw new CorruptIndexException("sorted entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
if (meta.readByte() != Lucene410DocValuesFormat.BINARY) {
- throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ throw new CorruptIndexException("sorted entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
BinaryEntry b = readBinaryEntry(meta);
- binaries.put(fieldNumber, b);
+ binaries.put(info.name, b);
- if (meta.readVInt() != fieldNumber) {
- throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ if (meta.readVInt() != info.number) {
+ throw new CorruptIndexException("sorted entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
if (meta.readByte() != Lucene410DocValuesFormat.NUMERIC) {
- throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ throw new CorruptIndexException("sorted entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
NumericEntry n = readNumericEntry(meta);
- ords.put(fieldNumber, n);
+ ords.put(info.name, n);
}
- private void readSortedSetFieldWithAddresses(int fieldNumber, IndexInput meta, FieldInfos infos) throws IOException {
+ private void readSortedSetFieldWithAddresses(FieldInfo info, IndexInput meta) throws IOException {
// sortedset = binary + numeric (addresses) + ordIndex
- if (meta.readVInt() != fieldNumber) {
- throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ if (meta.readVInt() != info.number) {
+ throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
if (meta.readByte() != Lucene410DocValuesFormat.BINARY) {
- throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
BinaryEntry b = readBinaryEntry(meta);
- binaries.put(fieldNumber, b);
+ binaries.put(info.name, b);
- if (meta.readVInt() != fieldNumber) {
- throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ if (meta.readVInt() != info.number) {
+ throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
if (meta.readByte() != Lucene410DocValuesFormat.NUMERIC) {
- throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
NumericEntry n1 = readNumericEntry(meta);
- ords.put(fieldNumber, n1);
+ ords.put(info.name, n1);
- if (meta.readVInt() != fieldNumber) {
- throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ if (meta.readVInt() != info.number) {
+ throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
if (meta.readByte() != Lucene410DocValuesFormat.NUMERIC) {
- throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
NumericEntry n2 = readNumericEntry(meta);
- ordIndexes.put(fieldNumber, n2);
+ ordIndexes.put(info.name, n2);
}
- private void readFields(IndexInput meta, FieldInfos infos) throws IOException {
+ private int readFields(IndexInput meta, FieldInfos infos) throws IOException {
+ int numFields = 0;
int fieldNumber = meta.readVInt();
while (fieldNumber != -1) {
- if (infos.fieldInfo(fieldNumber) == null) {
- // trickier to validate more: because we re-use for norms, because we use multiple entries
- // for "composite" types like sortedset, etc.
+ numFields++;
+ FieldInfo info = infos.fieldInfo(fieldNumber);
+ if (info == null) {
+ // trickier to validate more: because we use multiple entries for "composite" types like sortedset, etc.
throw new CorruptIndexException("Invalid field number: " + fieldNumber + " (resource=" + meta + ")");
}
byte type = meta.readByte();
if (type == Lucene410DocValuesFormat.NUMERIC) {
- numerics.put(fieldNumber, readNumericEntry(meta));
+ numerics.put(info.name, readNumericEntry(meta));
} else if (type == Lucene410DocValuesFormat.BINARY) {
BinaryEntry b = readBinaryEntry(meta);
- binaries.put(fieldNumber, b);
+ binaries.put(info.name, b);
} else if (type == Lucene410DocValuesFormat.SORTED) {
- readSortedField(fieldNumber, meta, infos);
+ readSortedField(info, meta);
} else if (type == Lucene410DocValuesFormat.SORTED_SET) {
SortedSetEntry ss = readSortedSetEntry(meta);
- sortedSets.put(fieldNumber, ss);
+ sortedSets.put(info.name, ss);
if (ss.format == SORTED_WITH_ADDRESSES) {
- readSortedSetFieldWithAddresses(fieldNumber, meta, infos);
+ readSortedSetFieldWithAddresses(info, meta);
} else if (ss.format == SORTED_SINGLE_VALUED) {
if (meta.readVInt() != fieldNumber) {
- throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
if (meta.readByte() != Lucene410DocValuesFormat.SORTED) {
- throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ throw new CorruptIndexException("sortedset entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
- readSortedField(fieldNumber, meta, infos);
+ readSortedField(info, meta);
} else {
throw new AssertionError();
}
} else if (type == Lucene410DocValuesFormat.SORTED_NUMERIC) {
SortedSetEntry ss = readSortedSetEntry(meta);
- sortedNumerics.put(fieldNumber, ss);
+ sortedNumerics.put(info.name, ss);
if (meta.readVInt() != fieldNumber) {
- throw new CorruptIndexException("sortednumeric entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ throw new CorruptIndexException("sortednumeric entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
if (meta.readByte() != Lucene410DocValuesFormat.NUMERIC) {
- throw new CorruptIndexException("sortednumeric entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ throw new CorruptIndexException("sortednumeric entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
- numerics.put(fieldNumber, readNumericEntry(meta));
+ numerics.put(info.name, readNumericEntry(meta));
if (ss.format == SORTED_WITH_ADDRESSES) {
if (meta.readVInt() != fieldNumber) {
- throw new CorruptIndexException("sortednumeric entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ throw new CorruptIndexException("sortednumeric entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
if (meta.readByte() != Lucene410DocValuesFormat.NUMERIC) {
- throw new CorruptIndexException("sortednumeric entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
+ throw new CorruptIndexException("sortednumeric entry for field: " + info.name + " is corrupt (resource=" + meta + ")");
}
NumericEntry ordIndex = readNumericEntry(meta);
- ordIndexes.put(fieldNumber, ordIndex);
+ ordIndexes.put(info.name, ordIndex);
} else if (ss.format != SORTED_SINGLE_VALUED) {
throw new AssertionError();
}
@@ -253,6 +261,7 @@ class Lucene410DocValuesProducer extends
}
fieldNumber = meta.readVInt();
}
+ return numFields;
}
static NumericEntry readNumericEntry(IndexInput meta) throws IOException {
@@ -332,7 +341,7 @@ class Lucene410DocValuesProducer extends
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
- NumericEntry entry = numerics.get(field.number);
+ NumericEntry entry = numerics.get(field.name);
return getNumeric(entry);
}
@@ -342,10 +351,24 @@ class Lucene410DocValuesProducer extends
}
@Override
+ public synchronized Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ resources.addAll(Accountables.namedAccountables("addresses field", addressInstances));
+ resources.addAll(Accountables.namedAccountables("ord index field", ordIndexInstances));
+ resources.addAll(Accountables.namedAccountables("reverse index field", reverseIndexInstances));
+ return Collections.unmodifiableList(resources);
+ }
+
+ @Override
public void checkIntegrity() throws IOException {
CodecUtil.checksumEntireFile(data);
}
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(fields=" + numFields + ")";
+ }
+
LongValues getNumeric(NumericEntry entry) throws IOException {
RandomAccessInput slice = this.data.randomAccessSlice(entry.offset, entry.endOffset - entry.offset);
switch (entry.format) {
@@ -384,7 +407,7 @@ class Lucene410DocValuesProducer extends
@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
- BinaryEntry bytes = binaries.get(field.number);
+ BinaryEntry bytes = binaries.get(field.name);
switch(bytes.format) {
case BINARY_FIXED_UNCOMPRESSED:
return getFixedBinary(field, bytes);
@@ -420,11 +443,11 @@ class Lucene410DocValuesProducer extends
/** returns an address instance for variable-length binary values. */
private synchronized MonotonicBlockPackedReader getAddressInstance(FieldInfo field, BinaryEntry bytes) throws IOException {
- MonotonicBlockPackedReader addresses = addressInstances.get(field.number);
+ MonotonicBlockPackedReader addresses = addressInstances.get(field.name);
if (addresses == null) {
data.seek(bytes.addressesOffset);
addresses = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, bytes.count+1, false);
- addressInstances.put(field.number, addresses);
+ addressInstances.put(field.name, addresses);
ramBytesUsed.addAndGet(addresses.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
}
return addresses;
@@ -457,12 +480,12 @@ class Lucene410DocValuesProducer extends
/** returns an address instance for prefix-compressed binary values. */
private synchronized MonotonicBlockPackedReader getIntervalInstance(FieldInfo field, BinaryEntry bytes) throws IOException {
- MonotonicBlockPackedReader addresses = addressInstances.get(field.number);
+ MonotonicBlockPackedReader addresses = addressInstances.get(field.name);
if (addresses == null) {
data.seek(bytes.addressesOffset);
final long size = (bytes.count + INTERVAL_MASK) >>> INTERVAL_SHIFT;
addresses = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, size, false);
- addressInstances.put(field.number, addresses);
+ addressInstances.put(field.name, addresses);
ramBytesUsed.addAndGet(addresses.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
}
return addresses;
@@ -470,7 +493,7 @@ class Lucene410DocValuesProducer extends
/** returns a reverse lookup instance for prefix-compressed binary values. */
private synchronized ReverseTermsIndex getReverseIndexInstance(FieldInfo field, BinaryEntry bytes) throws IOException {
- ReverseTermsIndex index = reverseIndexInstances.get(field.number);
+ ReverseTermsIndex index = reverseIndexInstances.get(field.name);
if (index == null) {
index = new ReverseTermsIndex();
data.seek(bytes.reverseIndexOffset);
@@ -480,8 +503,8 @@ class Lucene410DocValuesProducer extends
PagedBytes pagedBytes = new PagedBytes(15);
pagedBytes.copy(data, dataSize);
index.terms = pagedBytes.freeze(true);
- reverseIndexInstances.put(field.number, index);
- ramBytesUsed.addAndGet(index.termAddresses.ramBytesUsed() + index.terms.ramBytesUsed());
+ reverseIndexInstances.put(field.name, index);
+ ramBytesUsed.addAndGet(index.ramBytesUsed());
}
return index;
}
@@ -496,9 +519,9 @@ class Lucene410DocValuesProducer extends
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
- final int valueCount = (int) binaries.get(field.number).count;
+ final int valueCount = (int) binaries.get(field.name).count;
final BinaryDocValues binary = getBinary(field);
- NumericEntry entry = ords.get(field.number);
+ NumericEntry entry = ords.get(field.name);
final LongValues ordinals = getNumeric(entry);
return new SortedDocValues() {
@@ -539,11 +562,11 @@ class Lucene410DocValuesProducer extends
/** returns an address instance for sortedset ordinal lists */
private synchronized MonotonicBlockPackedReader getOrdIndexInstance(FieldInfo field, NumericEntry entry) throws IOException {
- MonotonicBlockPackedReader instance = ordIndexInstances.get(field.number);
+ MonotonicBlockPackedReader instance = ordIndexInstances.get(field.name);
if (instance == null) {
data.seek(entry.offset);
instance = MonotonicBlockPackedReader.of(data, entry.packedIntsVersion, entry.blockSize, entry.count+1, false);
- ordIndexInstances.put(field.number, instance);
+ ordIndexInstances.put(field.name, instance);
ramBytesUsed.addAndGet(instance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
}
return instance;
@@ -551,14 +574,14 @@ class Lucene410DocValuesProducer extends
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
- SortedSetEntry ss = sortedNumerics.get(field.number);
- NumericEntry numericEntry = numerics.get(field.number);
+ SortedSetEntry ss = sortedNumerics.get(field.name);
+ NumericEntry numericEntry = numerics.get(field.name);
final LongValues values = getNumeric(numericEntry);
if (ss.format == SORTED_SINGLE_VALUED) {
final Bits docsWithField = getMissingBits(numericEntry.missingOffset);
return DocValues.singleton(values, docsWithField);
} else if (ss.format == SORTED_WITH_ADDRESSES) {
- final MonotonicBlockPackedReader ordIndex = getOrdIndexInstance(field, ordIndexes.get(field.number));
+ final MonotonicBlockPackedReader ordIndex = getOrdIndexInstance(field, ordIndexes.get(field.name));
return new SortedNumericDocValues() {
long startOffset;
@@ -587,7 +610,7 @@ class Lucene410DocValuesProducer extends
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
- SortedSetEntry ss = sortedSets.get(field.number);
+ SortedSetEntry ss = sortedSets.get(field.name);
if (ss.format == SORTED_SINGLE_VALUED) {
final SortedDocValues values = getSorted(field);
return DocValues.singleton(values);
@@ -595,12 +618,12 @@ class Lucene410DocValuesProducer extends
throw new AssertionError();
}
- final long valueCount = binaries.get(field.number).count;
+ final long valueCount = binaries.get(field.name).count;
// we keep the byte[]s and list of ords on disk, these could be large
final LongBinaryDocValues binary = (LongBinaryDocValues) getBinary(field);
- final LongValues ordinals = getNumeric(ords.get(field.number));
+ final LongValues ordinals = getNumeric(ords.get(field.name));
// but the addresses to the ord stream are in RAM
- final MonotonicBlockPackedReader ordIndex = getOrdIndexInstance(field, ordIndexes.get(field.number));
+ final MonotonicBlockPackedReader ordIndex = getOrdIndexInstance(field, ordIndexes.get(field.name));
return new RandomAccessOrds() {
long startOffset;
@@ -698,10 +721,10 @@ class Lucene410DocValuesProducer extends
case SORTED:
return DocValues.docsWithValue(getSorted(field), maxDoc);
case BINARY:
- BinaryEntry be = binaries.get(field.number);
+ BinaryEntry be = binaries.get(field.name);
return getMissingBits(be.missingOffset);
case NUMERIC:
- NumericEntry ne = numerics.get(field.number);
+ NumericEntry ne = numerics.get(field.name);
return getMissingBits(ne.missingOffset);
default:
throw new AssertionError();
@@ -778,9 +801,27 @@ class Lucene410DocValuesProducer extends
}
// used for reverse lookup to a small range of blocks
- static class ReverseTermsIndex {
+ static class ReverseTermsIndex implements Accountable {
public MonotonicBlockPackedReader termAddresses;
public PagedBytes.Reader terms;
+
+ @Override
+ public long ramBytesUsed() {
+ return termAddresses.ramBytesUsed() + terms.ramBytesUsed();
+ }
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ resources.add(Accountables.namedAccountable("term bytes", terms));
+ resources.add(Accountables.namedAccountable("term addresses", termAddresses));
+ return Collections.unmodifiableList(resources);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(size=" + termAddresses.size() + ")";
+ }
}
//in the compressed case, we add a few additional operations for
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java Tue Sep 16 13:26:55 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.codecs.CodecUtil;
@@ -32,6 +33,8 @@ import org.apache.lucene.index.NumericDo
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.BlockPackedReader;
@@ -49,15 +52,17 @@ import static org.apache.lucene.codecs.l
*/
class Lucene49NormsProducer extends NormsProducer {
// metadata maps (just file pointers and minimal stuff)
- private final Map<Integer,NormsEntry> norms = new HashMap<>();
+ private final Map<String,NormsEntry> norms = new HashMap<>();
private final IndexInput data;
private final int version;
// ram instances we have already loaded
- final Map<Integer,NumericDocValues> instances = new HashMap<>();
+ final Map<String,NumericDocValues> instances = new HashMap<>();
+ final Map<String,Accountable> instancesInfo = new HashMap<>();
private final int maxDoc;
private final AtomicLong ramBytesUsed;
+ private final AtomicInteger activeCount = new AtomicInteger();
Lucene49NormsProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
maxDoc = state.segmentInfo.getDocCount();
@@ -123,17 +128,18 @@ class Lucene49NormsProducer extends Norm
default:
throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
}
- norms.put(fieldNumber, entry);
+ norms.put(info.name, entry);
fieldNumber = meta.readVInt();
}
}
@Override
public synchronized NumericDocValues getNorms(FieldInfo field) throws IOException {
- NumericDocValues instance = instances.get(field.number);
+ NumericDocValues instance = instances.get(field.name);
if (instance == null) {
instance = loadNorms(field);
- instances.put(field.number, instance);
+ instances.put(field.name, instance);
+ activeCount.incrementAndGet();
}
return instance;
}
@@ -144,14 +150,21 @@ class Lucene49NormsProducer extends Norm
}
@Override
+ public synchronized Iterable<? extends Accountable> getChildResources() {
+ return Accountables.namedAccountables("field", instancesInfo);
+ }
+
+ @Override
public void checkIntegrity() throws IOException {
CodecUtil.checksumEntireFile(data);
}
private NumericDocValues loadNorms(FieldInfo field) throws IOException {
- NormsEntry entry = norms.get(field.number);
+ NormsEntry entry = norms.get(field.name);
switch(entry.format) {
case CONST_COMPRESSED:
+ instancesInfo.put(field.name, Accountables.namedAccountable("constant", 8));
+ ramBytesUsed.addAndGet(8);
final long v = entry.offset;
return new NumericDocValues() {
@Override
@@ -164,6 +177,7 @@ class Lucene49NormsProducer extends Norm
final byte bytes[] = new byte[maxDoc];
data.readBytes(bytes, 0, bytes.length);
ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes));
+ instancesInfo.put(field.name, Accountables.namedAccountable("byte array", maxDoc));
return new NumericDocValues() {
@Override
public long get(int docID) {
@@ -176,6 +190,7 @@ class Lucene49NormsProducer extends Norm
int blockSize = data.readVInt();
final BlockPackedReader reader = new BlockPackedReader(data, packedIntsVersion, blockSize, maxDoc, false);
ramBytesUsed.addAndGet(reader.ramBytesUsed());
+ instancesInfo.put(field.name, Accountables.namedAccountable("delta compressed", reader));
return reader;
case TABLE_COMPRESSED:
data.seek(entry.offset);
@@ -192,6 +207,7 @@ class Lucene49NormsProducer extends Norm
final int bitsPerValue = data.readVInt();
final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), packedVersion, maxDoc, bitsPerValue);
ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed());
+ instancesInfo.put(field.name, Accountables.namedAccountable("table compressed", ordsReader));
return new NumericDocValues() {
@Override
public long get(int docID) {
@@ -212,4 +228,9 @@ class Lucene49NormsProducer extends Norm
byte format;
long offset;
}
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(fields=" + norms.size() + ",active=" + activeCount.get() + ")";
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java Tue Sep 16 13:26:55 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs.perfiel
import java.io.Closeable;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Map;
@@ -37,6 +38,8 @@ import org.apache.lucene.index.SegmentWr
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -322,6 +325,11 @@ public abstract class PerFieldDocValuesF
}
return size;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Accountables.namedAccountables("format", formats);
+ }
@Override
public void checkIntegrity() throws IOException {
@@ -329,6 +337,11 @@ public abstract class PerFieldDocValuesF
format.checkIntegrity();
}
}
+
+ @Override
+ public String toString() {
+ return "PerFieldDocValues(formats=" + formats.size() + ")";
+ }
}
@Override
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java Tue Sep 16 13:26:55 2014
@@ -38,6 +38,8 @@ import org.apache.lucene.index.Fields;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
@@ -268,6 +270,11 @@ public abstract class PerFieldPostingsFo
}
return ramBytesUsed;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Accountables.namedAccountables("format", formats);
+ }
@Override
public void checkIntegrity() throws IOException {
@@ -275,6 +282,11 @@ public abstract class PerFieldPostingsFo
producer.checkIntegrity();
}
}
+
+ @Override
+ public String toString() {
+ return "PerFieldPostings(formats=" + formats.size() + ")";
+ }
}
@Override
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java Tue Sep 16 13:26:55 2014
@@ -116,6 +116,11 @@ class BufferedUpdatesStream implements A
public long ramBytesUsed() {
return bytesUsed.get();
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
public static class ApplyDeletesResult {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Tue Sep 16 13:26:55 2014
@@ -39,6 +39,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -532,7 +533,7 @@ public class CheckIndex {
int toLoseDocCount = info.info.getDocCount();
- AtomicReader reader = null;
+ SegmentReader reader = null;
try {
msg(infoStream, " version=" + (version == null ? "3.0" : version));
@@ -660,6 +661,11 @@ public class CheckIndex {
}
msg(infoStream, "");
+
+ if (verbose) {
+ msg(infoStream, "detailed segment RAM usage: ");
+ msg(infoStream, Accountables.toString(reader));
+ }
} catch (Throwable t) {
if (failFast) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Tue Sep 16 13:26:55 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.IOException;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Queue;
import java.util.Set;
@@ -669,6 +670,11 @@ final class DocumentsWriter implements C
public long ramBytesUsed() {
return flushControl.ramBytesUsed();
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
static final class ApplyDeletesEvent implements Event {
static final Event INSTANCE = new ApplyDeletesEvent();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java Tue Sep 16 13:26:55 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.util.Arrays;
+import java.util.Collections;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import java.util.concurrent.locks.ReentrantLock;
@@ -456,6 +457,11 @@ final class DocumentsWriterDeleteQueue i
}
@Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+
+ @Override
public String toString() {
return "DWDQ: [ generation: " + generation + " ]";
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java Tue Sep 16 13:26:55 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.util.ArrayList;
+import java.util.Collections;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.LinkedList;
@@ -437,6 +438,12 @@ final class DocumentsWriterFlushControl
return getDeleteBytesUsed() + netBytes();
}
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ // TODO: improve this?
+ return Collections.emptyList();
+ }
+
synchronized int numFlushingDWPT() {
return flushingWriters.size();
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Tue Sep 16 13:26:55 2014
@@ -463,6 +463,11 @@ public class IndexWriter implements Clos
ensureOpen();
return docWriter.ramBytesUsed();
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
/** Holds shared SegmentReader instances. IndexWriter uses
* SegmentReaders for 1) applying deletes, 2) doing
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java Tue Sep 16 13:26:55 2014
@@ -18,12 +18,15 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import org.apache.lucene.index.MultiTermsEnum.TermsEnumIndex;
import org.apache.lucene.index.MultiTermsEnum.TermsEnumWithSlice;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.InPlaceMergeSorter;
@@ -429,6 +432,10 @@ public class MultiDocValues {
return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(newToOld) + RamUsageEstimator.sizeOf(oldToNew);
}
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
}
/**
@@ -634,6 +641,16 @@ public class MultiDocValues {
public long ramBytesUsed() {
return ramBytesUsed;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ resources.add(Accountables.namedAccountable("global ord deltas", globalOrdDeltas));
+ resources.add(Accountables.namedAccountable("first segments", firstSegments));
+ resources.add(Accountables.namedAccountable("segment map", segmentMap));
+ // TODO: would be nice to return actual child segment deltas too, but the optimizations are confusing
+ return resources;
+ }
}
/**
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/PrefixCodedTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/PrefixCodedTerms.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/PrefixCodedTerms.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/PrefixCodedTerms.java Tue Sep 16 13:26:55 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.Collections;
import java.util.Iterator;
import org.apache.lucene.store.IndexInput;
@@ -44,6 +45,11 @@ class PrefixCodedTerms implements Iterab
return buffer.ramBytesUsed();
}
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+
/** @return iterator over the bytes */
@Override
public Iterator<Term> iterator() {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java Tue Sep 16 13:26:55 2014
@@ -215,6 +215,7 @@ final class SegmentCoreReaders implement
coreClosedListeners.remove(listener);
}
+ // TODO: remove this, it can just be on SR
@Override
public long ramBytesUsed() {
return BASE_RAM_BYTES_USED +
@@ -223,4 +224,9 @@ final class SegmentCoreReaders implement
((fieldsReaderOrig!=null)? fieldsReaderOrig.ramBytesUsed() : 0) +
((termVectorsReaderOrig!=null) ? termVectorsReaderOrig.ramBytesUsed() : 0);
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java Tue Sep 16 13:26:55 2014
@@ -30,6 +30,8 @@ import org.apache.lucene.codecs.DocValue
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.Version;
@@ -185,5 +187,19 @@ class SegmentDocValuesProducer extends D
ramBytesUsed += producer.ramBytesUsed();
}
return ramBytesUsed;
- }
+ }
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ for (Accountable producer : dvProducers) {
+ resources.add(Accountables.namedAccountable("delegate", producer));
+ }
+ return Collections.unmodifiableList(resources);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(producers=" + dvProducers.size() + ")";
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Tue Sep 16 13:26:55 2014
@@ -18,14 +18,17 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
@@ -34,6 +37,7 @@ import org.apache.lucene.store.CompoundF
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.CloseableThreadLocal;
import org.apache.lucene.util.IOUtils;
@@ -258,7 +262,7 @@ public final class SegmentReader extends
}
@Override
- public Fields fields() {
+ public FieldsProducer fields() {
ensureOpen();
return core.fields;
}
@@ -538,6 +542,28 @@ public final class SegmentReader extends
}
@Override
+ public Iterable<? extends Accountable> getChildResources() {
+ ensureOpen();
+ List<Accountable> resources = new ArrayList<>();
+ if (core.fields != null) {
+ resources.add(Accountables.namedAccountable("postings", core.fields));
+ }
+ if (core.normsProducer != null) {
+ resources.add(Accountables.namedAccountable("norms", core.normsProducer));
+ }
+ if (docValuesProducer != null) {
+ resources.add(Accountables.namedAccountable("docvalues", docValuesProducer));
+ }
+ if (getFieldsReader() != null) {
+ resources.add(Accountables.namedAccountable("stored fields", getFieldsReader()));
+ }
+ if (getTermVectorsReader() != null) {
+ resources.add(Accountables.namedAccountable("term vectors", getTermVectorsReader()));
+ }
+ return resources;
+ }
+
+ @Override
public void checkIntegrity() throws IOException {
ensureOpen();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java Tue Sep 16 13:26:55 2014
@@ -29,6 +29,7 @@ import java.util.WeakHashMap;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.WAH8DocIdSet;
@@ -147,4 +148,10 @@ public class CachingWrapperFilter extend
return total;
}
+
+ @Override
+ public synchronized Iterable<? extends Accountable> getChildResources() {
+ // Sync only to pull the current set of values:
+ return Accountables.namedAccountables("segment", cache);
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/DocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/DocIdSet.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/DocIdSet.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/DocIdSet.java Tue Sep 16 13:26:55 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+import java.util.Collections;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
@@ -93,4 +94,8 @@ public abstract class DocIdSet implement
return false;
}
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java Tue Sep 16 13:26:55 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
import java.io.IOException;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.RamUsageEstimator;
@@ -62,6 +63,11 @@ public abstract class FilteredDocIdSet e
public long ramBytesUsed() {
return RamUsageEstimator.NUM_BYTES_OBJECT_REF + _innerSet.ramBytesUsed();
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return _innerSet.getChildResources();
+ }
@Override
public Bits bits() throws IOException {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java Tue Sep 16 13:26:55 2014
@@ -20,11 +20,13 @@ package org.apache.lucene.store;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.apache.lucene.store.RAMDirectory; // javadocs
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.IOUtils;
// TODO
@@ -256,4 +258,9 @@ public class NRTCachingDirectory extends
public long ramBytesUsed() {
return cache.ramBytesUsed();
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.singleton(Accountables.namedAccountable("cache", cache));
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java Tue Sep 16 13:26:55 2014
@@ -28,6 +28,7 @@ import java.util.concurrent.ConcurrentHa
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
/**
@@ -150,6 +151,16 @@ public class RAMDirectory extends BaseDi
return sizeInBytes.get();
}
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Accountables.namedAccountables("file", fileMap);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(id=" + getLockID() + ")";
+ }
+
/** Removes an existing file in the directory.
* @throws IOException if the file does not exist
*/
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMFile.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMFile.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMFile.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMFile.java Tue Sep 16 13:26:55 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.store;
*/
import java.util.ArrayList;
+import java.util.Collections;
import org.apache.lucene.util.Accountable;
@@ -82,4 +83,13 @@ public class RAMFile implements Accounta
return sizeInBytes;
}
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(length=" + length + ")";
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java Tue Sep 16 13:26:55 2014
@@ -18,10 +18,12 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
+import java.util.Collections;
import java.util.zip.CRC32;
import java.util.zip.Checksum;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
/**
* A memory-resident {@link IndexOutput} implementation.
@@ -182,6 +184,11 @@ public class RAMOutputStream extends Ind
public long ramBytesUsed() {
return (long) file.numBuffers() * (long) BUFFER_SIZE;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.singleton(Accountables.namedAccountable("file", file));
+ }
@Override
public long getChecksum() throws IOException {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/Accountable.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/Accountable.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/Accountable.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/Accountable.java Tue Sep 16 13:26:55 2014
@@ -29,4 +29,12 @@ public interface Accountable {
*/
long ramBytesUsed();
+ /**
+ * Returns nested resources of this class.
+ * The result should be a point-in-time snapshot (to avoid race conditions).
+ * @see Accountables
+ */
+ // TODO: on java8 make this a default method returning emptyList
+ Iterable<? extends Accountable> getChildResources();
+
}
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/Accountables.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/Accountables.java?rev=1625275&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/Accountables.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/Accountables.java Tue Sep 16 13:26:55 2014
@@ -0,0 +1,141 @@
+package org.apache.lucene.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Helper methods for constructing nested resource descriptions
+ * and debugging RAM usage.
+ * <p>
+ * {@code toString(Accountable}} can be used to quickly debug the nested
+ * structure of any Accountable.
+ * <p>
+ * The {@code namedAccountable} and {@code namedAccountables} methods return
+ * type-safe, point-in-time snapshots of the provided resources.
+ */
+public class Accountables {
+ private Accountables() {}
+
+ /**
+ * Returns a String description of an Accountable and any nested resources.
+ * This is intended for development and debugging.
+ */
+ public static String toString(Accountable a) {
+ StringBuilder sb = new StringBuilder();
+ toString(sb, a, 0);
+ return sb.toString();
+ }
+
+ private static StringBuilder toString(StringBuilder dest, Accountable a, int depth) {
+ for (int i = 1; i < depth; i++) {
+ dest.append(" ");
+ }
+
+ if (depth > 0) {
+ dest.append("|-- ");
+ }
+
+ dest.append(a.toString());
+ dest.append(": ");
+ dest.append(RamUsageEstimator.humanReadableUnits(a.ramBytesUsed()));
+ dest.append(System.lineSeparator());
+
+ for (Accountable child : a.getChildResources()) {
+ toString(dest, child, depth + 1);
+ }
+
+ return dest;
+ }
+
+ /**
+ * Augments an existing accountable with the provided description.
+ * <p>
+ * The resource description is constructed in this format:
+ * {@code description [toString()]}
+ * <p>
+ * This is a point-in-time type safe view: consumers
+ * will not be able to cast or manipulate the resource in any way.
+ */
+ public static Accountable namedAccountable(String description, Accountable in) {
+ return namedAccountable(description + " [" + in + "]", in.getChildResources(), in.ramBytesUsed());
+ }
+
+ /**
+ * Returns an accountable with the provided description and bytes.
+ */
+ public static Accountable namedAccountable(String description, long bytes) {
+ return namedAccountable(description, Collections.<Accountable>emptyList(), bytes);
+ }
+
+ /**
+ * Converts a map of resources to a collection.
+ * <p>
+ * The resource descriptions are constructed in this format:
+ * {@code prefix 'key' [toString()]}
+ * <p>
+ * This is a point-in-time type safe view: consumers
+ * will not be able to cast or manipulate the resources in any way.
+ */
+ public static Collection<Accountable> namedAccountables(String prefix, Map<?,? extends Accountable> in) {
+ List<Accountable> resources = new ArrayList<>();
+ for (Map.Entry<?,? extends Accountable> kv : in.entrySet()) {
+ resources.add(namedAccountable(prefix + " '" + kv.getKey() + "'", kv.getValue()));
+ }
+ Collections.sort(resources, new Comparator<Accountable>() {
+ @Override
+ public int compare(Accountable o1, Accountable o2) {
+ return o1.toString().compareTo(o2.toString());
+ }
+ });
+ return Collections.unmodifiableList(resources);
+ }
+
+ /**
+ * Returns an accountable with the provided description, bytes, and children.
+ * <p>
+ * The resource descriptions are constructed in this format:
+ * {@code description [toString()]}
+ * <p>
+ * This is a point-in-time type safe view: consumers
+ * will not be able to cast or manipulate the resources in any way..
+ */
+ private static Accountable namedAccountable(final String description, final Iterable<? extends Accountable> children, final long bytes) {
+ return new Accountable() {
+ @Override
+ public long ramBytesUsed() {
+ return bytes;
+ }
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return children;
+ }
+
+ @Override
+ public String toString() {
+ return description;
+ }
+ };
+ }
+}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java Tue Sep 16 13:26:55 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.util;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
@@ -134,6 +135,16 @@ public final class PagedBytes implements
}
return size;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public String toString() {
+ return "PagedBytes(blocksize=" + blockSize + ")";
+ }
}
/** 1<<blockBits must be bigger than biggest single
@@ -246,6 +257,11 @@ public final class PagedBytes implements
}
return size;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
/** Copy bytes in, writing the length as a 1 or 2 byte
* vInt prefix. */
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java Tue Sep 16 13:26:55 2014
@@ -157,4 +157,9 @@ public final class ByteSequenceOutputs e
public long ramBytesUsed(BytesRef output) {
return BASE_NUM_BYTES + RamUsageEstimator.sizeOf(output.bytes);
}
+
+ @Override
+ public String toString() {
+ return "ByteSequenceOutputs";
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java Tue Sep 16 13:26:55 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.util.fst;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import org.apache.lucene.store.DataInput;
@@ -480,5 +481,14 @@ class BytesStore extends DataOutput impl
}
return size;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(numBlocks=" + blocks.size() + ")";
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/FST.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/FST.java Tue Sep 16 13:26:55 2014
@@ -22,9 +22,11 @@ import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.ArrayList;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
@@ -35,6 +37,7 @@ import org.apache.lucene.store.InputStre
import org.apache.lucene.store.OutputStreamDataOutput;
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
@@ -444,6 +447,23 @@ public final class FST<T> implements Acc
return size;
}
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ if (packed) {
+ resources.add(Accountables.namedAccountable("node ref to address", nodeRefToAddress));
+ } else if (nodeAddress != null) {
+ resources.add(Accountables.namedAccountable("node addresses", nodeAddress));
+ resources.add(Accountables.namedAccountable("in counts", inCounts));
+ }
+ return resources;
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(input=" + inputType + ",output=" + outputs + ",packed=" + packed + ",nodes=" + nodeCount + ",arcs=" + arcCount + ")";
+ }
+
void finish(long newStartNode) throws IOException {
if (startNode != -1) {
throw new IllegalStateException("already finished");
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java Tue Sep 16 13:26:55 2014
@@ -160,4 +160,9 @@ public final class IntSequenceOutputs ex
public long ramBytesUsed(IntsRef output) {
return BASE_NUM_BYTES + RamUsageEstimator.sizeOf(output.ints);
}
+
+ @Override
+ public String toString() {
+ return "IntSequenceOutputs";
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/NoOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/NoOutputs.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/NoOutputs.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/NoOutputs.java Tue Sep 16 13:26:55 2014
@@ -106,4 +106,9 @@ public final class NoOutputs extends Out
public long ramBytesUsed(Object output) {
return 0;
}
+
+ @Override
+ public String toString() {
+ return "NoOutputs";
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java Tue Sep 16 13:26:55 2014
@@ -27,6 +27,7 @@ import static org.apache.lucene.util.pac
import static org.apache.lucene.util.packed.PackedInts.numBlocks;
import java.io.IOException;
+import java.util.Collections;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
@@ -42,6 +43,7 @@ public final class BlockPackedReader ext
private final long valueCount;
private final long[] minValues;
private final PackedInts.Reader[] subReaders;
+ private final long sumBPV;
/** Sole constructor. */
public BlockPackedReader(IndexInput in, int packedIntsVersion, int blockSize, long valueCount, boolean direct) throws IOException {
@@ -51,9 +53,11 @@ public final class BlockPackedReader ext
final int numBlocks = numBlocks(valueCount, blockSize);
long[] minValues = null;
subReaders = new PackedInts.Reader[numBlocks];
+ long sumBPV = 0;
for (int i = 0; i < numBlocks; ++i) {
final int token = in.readByte() & 0xFF;
final int bitsPerValue = token >>> BPV_SHIFT;
+ sumBPV += bitsPerValue;
if (bitsPerValue > 64) {
throw new IOException("Corrupted");
}
@@ -77,6 +81,7 @@ public final class BlockPackedReader ext
}
}
this.minValues = minValues;
+ this.sumBPV = sumBPV;
}
@Override
@@ -95,4 +100,15 @@ public final class BlockPackedReader ext
}
return size;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public String toString() {
+ long avgBPV = subReaders.length == 0 ? 0 : sumBPV / subReaders.length;
+ return getClass().getSimpleName() + "(blocksize=" + (1<<blockShift) + ",size=" + valueCount + ",avgBPV=" + avgBPV + ")";
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoEncoder.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoEncoder.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoEncoder.java Tue Sep 16 13:26:55 2014
@@ -18,6 +18,7 @@
package org.apache.lucene.util.packed;
import java.util.Arrays;
+import java.util.Collections;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.FixedBitSet; // for javadocs
@@ -362,5 +363,10 @@ public class EliasFanoEncoder implements
+ RamUsageEstimator.sizeOf(upperLongs)
+ RamUsageEstimator.sizeOf(upperZeroBitPositionIndex);
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java Tue Sep 16 13:26:55 2014
@@ -24,6 +24,7 @@ import static org.apache.lucene.util.pac
import static org.apache.lucene.util.packed.PackedInts.numBlocks;
import java.io.IOException;
+import java.util.Collections;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
@@ -46,6 +47,7 @@ public class MonotonicBlockPackedReader
final long[] minValues;
final float[] averages;
final PackedInts.Reader[] subReaders;
+ final long sumBPV;
/** Sole constructor. */
public static MonotonicBlockPackedReader of(IndexInput in, int packedIntsVersion, int blockSize, long valueCount, boolean direct) throws IOException {
@@ -68,6 +70,7 @@ public class MonotonicBlockPackedReader
minValues = new long[numBlocks];
averages = new float[numBlocks];
subReaders = new PackedInts.Reader[numBlocks];
+ long sumBPV = 0;
for (int i = 0; i < numBlocks; ++i) {
if (packedIntsVersion < PackedInts.VERSION_MONOTONIC_WITHOUT_ZIGZAG) {
minValues[i] = in.readVLong();
@@ -76,6 +79,7 @@ public class MonotonicBlockPackedReader
}
averages[i] = Float.intBitsToFloat(in.readInt());
final int bitsPerValue = in.readVInt();
+ sumBPV += bitsPerValue;
if (bitsPerValue > 64) {
throw new IOException("Corrupted");
}
@@ -92,6 +96,7 @@ public class MonotonicBlockPackedReader
}
}
}
+ this.sumBPV = sumBPV;
}
@Override
@@ -121,5 +126,15 @@ public class MonotonicBlockPackedReader
}
return sizeInBytes;
}
-
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public String toString() {
+ long avgBPV = subReaders.length == 0 ? 0 : sumBPV / subReaders.length;
+ return getClass().getSimpleName() + "(blocksize=" + (1<<blockShift) + ",size=" + valueCount + ",avgBPV=" + avgBPV + ")";
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed16ThreeBlocks.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed16ThreeBlocks.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed16ThreeBlocks.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed16ThreeBlocks.java Tue Sep 16 13:26:55 2014
@@ -126,6 +126,6 @@ final class Packed16ThreeBlocks extends
@Override
public String toString() {
return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue
- + ", size=" + size() + ", elements.length=" + blocks.length + ")";
+ + ",size=" + size() + ",blocks=" + blocks.length + ")";
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java Tue Sep 16 13:26:55 2014
@@ -238,8 +238,8 @@ class Packed64 extends PackedInts.Mutabl
@Override
public String toString() {
- return "Packed64(bitsPerValue=" + bitsPerValue + ", size="
- + size() + ", elements.length=" + blocks.length + ")";
+ return "Packed64(bitsPerValue=" + bitsPerValue + ",size="
+ + size() + ",blocks=" + blocks.length + ")";
}
@Override
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java Tue Sep 16 13:26:55 2014
@@ -203,7 +203,7 @@ abstract class Packed64SingleBlock exten
@Override
public String toString() {
return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue
- + ", size=" + size() + ", elements.length=" + blocks.length + ")";
+ + ",size=" + size() + ",blocks=" + blocks.length + ")";
}
public static Packed64SingleBlock create(DataInput in,
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed8ThreeBlocks.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed8ThreeBlocks.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed8ThreeBlocks.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed8ThreeBlocks.java Tue Sep 16 13:26:55 2014
@@ -124,6 +124,6 @@ final class Packed8ThreeBlocks extends P
@Override
public String toString() {
return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue
- + ", size=" + size() + ", elements.length=" + blocks.length + ")";
+ + ",size=" + size() + ",blocks=" + blocks.length + ")";
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java Tue Sep 16 13:26:55 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.util.packed;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.NumericDocValues;
@@ -478,6 +479,10 @@ public class PackedInts {
*/
public abstract int size();
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
}
/**
@@ -627,7 +632,6 @@ public class PackedInts {
public final int size() {
return valueCount;
}
-
}
static abstract class MutableImpl extends Mutable {
@@ -651,6 +655,10 @@ public class PackedInts {
return valueCount;
}
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(valueCount=" + valueCount + ",bitsPerValue=" + bitsPerValue + ")";
+ }
}
/** A {@link Reader} which has all its values equal to 0 (bitsPerValue = 0). */
@@ -686,7 +694,6 @@ public class PackedInts {
public long ramBytesUsed() {
return RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT);
}
-
}
/** A write-once Writer.
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedLongValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedLongValues.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedLongValues.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedLongValues.java Tue Sep 16 13:26:55 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.util.packed;
import static org.apache.lucene.util.packed.PackedInts.checkBlockSize;
import java.util.Arrays;
+import java.util.Collections;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.ArrayUtil;
@@ -114,6 +115,11 @@ public class PackedLongValues extends Lo
public long ramBytesUsed() {
return ramBytesUsed;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
/** Return an iterator over the values of this array. */
public Iterator iterator() {
@@ -208,6 +214,11 @@ public class PackedLongValues extends Lo
return ramBytesUsed;
}
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+
/** Return the number of elements that have been added to this builder. */
public final long size() {
return size;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/gen_Packed64SingleBlock.py
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/gen_Packed64SingleBlock.py?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/gen_Packed64SingleBlock.py (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/gen_Packed64SingleBlock.py Tue Sep 16 13:26:55 2014
@@ -222,7 +222,7 @@ abstract class Packed64SingleBlock exten
@Override
public String toString() {
return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue
- + ", size=" + size() + ", elements.length=" + blocks.length + ")";
+ + ",size=" + size() + ",blocks=" + blocks.length + ")";
}
public static Packed64SingleBlock create(DataInput in,
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/gen_PackedThreeBlocks.py
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/gen_PackedThreeBlocks.py?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/gen_PackedThreeBlocks.py (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/gen_PackedThreeBlocks.py Tue Sep 16 13:26:55 2014
@@ -158,7 +158,7 @@ if __name__ == '__main__':
@Override
public String toString() {
return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue
- + ", size=" + size() + ", elements.length=" + blocks.length + ")";
+ + ",size=" + size() + ",blocks=" + blocks.length + ")";
}
}
""" %(MASKS[bpv], 2*bpv, MASKS[bpv], bpv, MASKS[bpv], MASKS[bpv], 2*bpv, MASKS[bpv], bpv, MASKS[bpv], CASTS[bpv], 2*bpv, CASTS[bpv], bpv, CASTS[bpv], CASTS[bpv],
Modified: lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CachedOrdinalsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CachedOrdinalsReader.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CachedOrdinalsReader.java (original)
+++ lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CachedOrdinalsReader.java Tue Sep 16 13:26:55 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.facet.taxonomy
*/
import java.io.IOException;
+import java.util.Collections;
import java.util.Map;
import java.util.WeakHashMap;
@@ -25,6 +26,7 @@ import org.apache.lucene.codecs.DocValue
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.RamUsageEstimator;
@@ -146,6 +148,11 @@ public class CachedOrdinalsReader extend
}
return mem;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
}
@Override
@@ -157,4 +164,9 @@ public class CachedOrdinalsReader extend
return bytes;
}
+
+ @Override
+ public synchronized Iterable<? extends Accountable> getChildResources() {
+ return Accountables.namedAccountables("segment", ordsCache);
+ }
}
Modified: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java (original)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java Tue Sep 16 13:26:55 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.uninverting;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import org.apache.lucene.codecs.PostingsFormat; // javadocs
@@ -181,6 +182,11 @@ public class DocTermOrds implements Acco
return sz;
}
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+
/** Inverts all terms */
public DocTermOrds(AtomicReader reader, Bits liveDocs, String field) throws IOException {
this(reader, liveDocs, field, null, Integer.MAX_VALUE);
Modified: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java?rev=1625275&r1=1625274&r2=1625275&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java (original)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java Tue Sep 16 13:26:55 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.uninverting;
import java.io.IOException;
import java.io.PrintStream;
+import java.util.Collections;
import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.lucene.document.DoubleField;
@@ -63,6 +64,11 @@ interface FieldCache {
// don't call on the in-progress value, might make things angry.
return RamUsageEstimator.NUM_BYTES_OBJECT_REF;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
}
/**