You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/09/16 21:00:52 UTC
svn commit: r1625356 [1/4] - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/
lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/
lucene/codecs/src/java/org/apache/lucen...
Author: rmuir
Date: Tue Sep 16 19:00:49 2014
New Revision: 1625356
URL: http://svn.apache.org/r1625356
Log:
LUCENE-5949: Add Accountable.getChildResources
Added:
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/Accountables.java
- copied unchanged from r1625275, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/Accountables.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash2.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextNormsFormat.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene410/Lucene410DocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/PrefixCodedTerms.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/DocIdSet.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/RAMFile.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/Accountable.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/NoOutputs.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/EliasFanoEncoder.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed16ThreeBlocks.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed8ThreeBlocks.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedLongValues.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/gen_Packed64SingleBlock.py
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/gen_PackedThreeBlocks.py
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CachedOrdinalsReader.java
lucene/dev/branches/branch_4x/lucene/misc/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java
lucene/dev/branches/branch_4x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java
lucene/dev/branches/branch_4x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionFieldReader.java
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/spelling/suggest/SolrSuggester.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Tue Sep 16 19:00:49 2014
@@ -36,6 +36,8 @@ New Features
* LUCENE-5820: SuggestStopFilter should have a factory.
(Varun Thacker via Steve Rowe)
+* LUCENE-5949: Add Accountable.getChildResources(). (Robert Muir)
+
API Changes:
* LUCENE-5900: Deprecated more constructors taking Version in *InfixSuggester and
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java Tue Sep 16 19:00:49 2014
@@ -18,9 +18,11 @@ package org.apache.lucene.codecs.blockte
*/
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
+import java.util.List;
import java.util.TreeMap;
import org.apache.lucene.codecs.BlockTermState;
@@ -43,6 +45,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -266,6 +269,11 @@ public class BlockTermsReader extends Fi
public long ramBytesUsed() {
return FIELD_READER_RAM_BYTES_USED;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
@Override
public Comparator<BytesRef> getComparator() {
@@ -898,6 +906,23 @@ public class BlockTermsReader extends Fi
}
return ramBytesUsed;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ if (indexReader != null) {
+ resources.add(Accountables.namedAccountable("term index", indexReader));
+ }
+ if (postingsReader != null) {
+ resources.add(Accountables.namedAccountable("delegate", postingsReader));
+ }
+ return Collections.unmodifiableList(resources);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(index=" + indexReader + ",delegate=" + postingsReader + ")";
+ }
@Override
public void checkIntegrity() throws IOException {
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java Tue Sep 16 19:00:49 2014
@@ -25,14 +25,18 @@ import org.apache.lucene.index.CorruptIn
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.PackedInts;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Comparator;
+import java.util.List;
import java.io.IOException;
import org.apache.lucene.index.IndexFileNames;
@@ -288,6 +292,20 @@ public class FixedGapTermsIndexReader ex
public long ramBytesUsed() {
return FIELD_INDEX_DATA_BASE_RAM_BYTES_USED + coreIndex.ramBytesUsed();
}
+
+ @Override
+ public String toString() {
+ return "FixedGapTermIndex(indexterms=" + numIndexTerms + ")";
+ }
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ if (coreIndex == null) {
+ return Collections.emptyList();
+ } else {
+ return Collections.singleton(Accountables.namedAccountable("term index", coreIndex));
+ }
+ }
private void loadTermsIndex(PagedBytes termBytes) throws IOException {
if (coreIndex == null) {
@@ -416,6 +434,23 @@ public class FixedGapTermsIndexReader ex
return ((termOffsets!=null)? termOffsets.ramBytesUsed() : 0) +
((termsDictOffsets!=null)? termsDictOffsets.ramBytesUsed() : 0);
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ if (termOffsets != null) {
+ resources.add(Accountables.namedAccountable("term lengths", termOffsets));
+ }
+ if (termsDictOffsets != null) {
+ resources.add(Accountables.namedAccountable("offsets", termsDictOffsets));
+ }
+ return Collections.unmodifiableList(resources);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(indexterms=" + numIndexTerms + ")";
+ }
}
}
@@ -457,4 +492,14 @@ public class FixedGapTermsIndexReader ex
}
return sizeInBytes;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Accountables.namedAccountables("field", fields);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(fields=" + fields.size() + ",interval=" + indexInterval + ")";
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java Tue Sep 16 19:00:49 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.blockte
*/
import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import org.apache.lucene.codecs.CodecUtil;
@@ -29,6 +30,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
@@ -223,6 +225,20 @@ public class VariableGapTermsIndexReader
public long ramBytesUsed() {
return fst == null ? 0 : fst.ramBytesUsed();
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ if (fst == null) {
+ return Collections.emptyList();
+ } else {
+ return Collections.singletonList(Accountables.namedAccountable("index data", fst));
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "VarGapTermIndex";
+ }
}
@Override
@@ -261,4 +277,14 @@ public class VariableGapTermsIndexReader
}
return sizeInBytes;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Accountables.namedAccountables("field", fields);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(fields=" + fields.size() + ")";
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java Tue Sep 16 19:00:49 2014
@@ -18,8 +18,10 @@ package org.apache.lucene.codecs.blocktr
*/
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import java.util.TreeMap;
import org.apache.lucene.codecs.CodecUtil;
@@ -36,6 +38,8 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -235,11 +239,19 @@ public final class OrdsBlockTreeTermsRea
@Override
public long ramBytesUsed() {
- long sizeInByes = ((postingsReader!=null) ? postingsReader.ramBytesUsed() : 0);
+ long sizeInBytes = postingsReader.ramBytesUsed();
for (OrdsFieldReader reader : fields.values()) {
- sizeInByes += reader.ramBytesUsed();
+ sizeInBytes += reader.ramBytesUsed();
}
- return sizeInByes;
+ return sizeInBytes;
+ }
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ resources.addAll(Accountables.namedAccountables("field", fields));
+ resources.add(Accountables.namedAccountable("delegate", postingsReader));
+ return Collections.unmodifiableList(resources);
}
@Override
@@ -250,4 +262,9 @@ public final class OrdsBlockTreeTermsRea
// postings
postingsReader.checkIntegrity();
}
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(fields=" + fields.size() + ",delegate=" + postingsReader.toString() + ")";
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java Tue Sep 16 19:00:49 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs.blocktr
import java.io.IOException;
import java.util.Comparator;
+import java.util.Collections;
import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output;
import org.apache.lucene.index.FieldInfo.IndexOptions;
@@ -28,6 +29,7 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.fst.FST;
@@ -176,4 +178,18 @@ final class OrdsFieldReader extends Term
public long ramBytesUsed() {
return ((index!=null)? index.ramBytesUsed() : 0);
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ if (index == null) {
+ return Collections.emptyList();
+ } else {
+ return Collections.singleton(Accountables.namedAccountable("term index", index));
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "OrdsBlockTreeTerms(terms=" + numTerms + ",postings=" + sumDocFreq + ",positions=" + sumTotalTermFreq + ",docs=" + docCount + ")";
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java Tue Sep 16 19:00:49 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.codecs.bloom;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -43,6 +44,8 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -424,9 +427,24 @@ public final class BloomFilteringPosting
}
@Override
+ public Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ resources.addAll(Accountables.namedAccountables("field", bloomsByFieldName));
+ if (delegateFieldsProducer != null) {
+ resources.add(Accountables.namedAccountable("delegate", delegateFieldsProducer));
+ }
+ return Collections.unmodifiableList(resources);
+ }
+
+ @Override
public void checkIntegrity() throws IOException {
delegateFieldsProducer.checkIntegrity();
}
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(fields=" + bloomsByFieldName.size() + ",delegate=" + delegateFieldsProducer + ")";
+ }
}
class BloomFilteredFieldsConsumer extends FieldsConsumer {
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java Tue Sep 16 19:00:49 2014
@@ -17,6 +17,7 @@ package org.apache.lucene.codecs.bloom;
* limitations under the License.
*/
import java.io.IOException;
+import java.util.Collections;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
@@ -309,4 +310,14 @@ public class FuzzySet implements Account
public long ramBytesUsed() {
return RamUsageEstimator.sizeOf(filter.getBits());
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(hash=" + hashFunction + ")";
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash2.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash2.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash2.java Tue Sep 16 19:00:49 2014
@@ -98,5 +98,9 @@ public final class MurmurHash2 extends H
public final int hash(BytesRef br) {
return hash32(br.bytes, br.offset, br.length);
}
-
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java Tue Sep 16 19:00:49 2014
@@ -18,7 +18,10 @@ package org.apache.lucene.codecs.memory;
*/
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
@@ -28,6 +31,7 @@ import org.apache.lucene.index.BinaryDoc
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.RandomAccessOrds;
@@ -37,6 +41,8 @@ import org.apache.lucene.index.SortedNum
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
@@ -49,25 +55,22 @@ import org.apache.lucene.util.RamUsageEs
class DirectDocValuesProducer extends DocValuesProducer {
// metadata maps (just file pointers and minimal stuff)
- private final Map<Integer,NumericEntry> numerics = new HashMap<>();
- private final Map<Integer,BinaryEntry> binaries = new HashMap<>();
- private final Map<Integer,SortedEntry> sorteds = new HashMap<>();
- private final Map<Integer,SortedSetEntry> sortedSets = new HashMap<>();
- private final Map<Integer,SortedNumericEntry> sortedNumerics = new HashMap<>();
+ private final Map<String,NumericEntry> numerics = new HashMap<>();
+ private final Map<String,BinaryEntry> binaries = new HashMap<>();
+ private final Map<String,SortedEntry> sorteds = new HashMap<>();
+ private final Map<String,SortedSetEntry> sortedSets = new HashMap<>();
+ private final Map<String,SortedNumericEntry> sortedNumerics = new HashMap<>();
private final IndexInput data;
// ram instances we have already loaded
- private final Map<Integer,NumericDocValues> numericInstances =
- new HashMap<>();
- private final Map<Integer,BinaryRawValues> binaryInstances =
- new HashMap<>();
- private final Map<Integer,SortedRawValues> sortedInstances =
- new HashMap<>();
- private final Map<Integer,SortedSetRawValues> sortedSetInstances =
- new HashMap<>();
- private final Map<Integer,SortedNumericRawValues> sortedNumericInstances =
- new HashMap<>();
- private final Map<Integer,Bits> docsWithFieldInstances = new HashMap<>();
+ private final Map<String,NumericRawValues> numericInstances = new HashMap<>();
+ private final Map<String,BinaryRawValues> binaryInstances = new HashMap<>();
+ private final Map<String,SortedRawValues> sortedInstances = new HashMap<>();
+ private final Map<String,SortedSetRawValues> sortedSetInstances = new HashMap<>();
+ private final Map<String,SortedNumericRawValues> sortedNumericInstances = new HashMap<>();
+ private final Map<String,FixedBitSet> docsWithFieldInstances = new HashMap<>();
+
+ private final int numEntries;
private final int maxDoc;
private final AtomicLong ramBytesUsed;
@@ -95,7 +98,7 @@ class DirectDocValuesProducer extends Do
version = CodecUtil.checkHeader(in, metaCodec,
VERSION_START,
VERSION_CURRENT);
- readFields(in);
+ numEntries = readFields(in, state.fieldInfos);
CodecUtil.checkFooter(in);
success = true;
@@ -188,37 +191,41 @@ class DirectDocValuesProducer extends Do
return entry;
}
- private void readFields(IndexInput meta) throws IOException {
+ private int readFields(IndexInput meta, FieldInfos infos) throws IOException {
+ int numEntries = 0;
int fieldNumber = meta.readVInt();
while (fieldNumber != -1) {
+ numEntries++;
+ FieldInfo info = infos.fieldInfo(fieldNumber);
int fieldType = meta.readByte();
if (fieldType == NUMBER) {
- numerics.put(fieldNumber, readNumericEntry(meta));
+ numerics.put(info.name, readNumericEntry(meta));
} else if (fieldType == BYTES) {
- binaries.put(fieldNumber, readBinaryEntry(meta));
+ binaries.put(info.name, readBinaryEntry(meta));
} else if (fieldType == SORTED) {
SortedEntry entry = readSortedEntry(meta);
- sorteds.put(fieldNumber, entry);
- binaries.put(fieldNumber, entry.values);
+ sorteds.put(info.name, entry);
+ binaries.put(info.name, entry.values);
} else if (fieldType == SORTED_SET) {
SortedSetEntry entry = readSortedSetEntry(meta, false);
- sortedSets.put(fieldNumber, entry);
- binaries.put(fieldNumber, entry.values);
+ sortedSets.put(info.name, entry);
+ binaries.put(info.name, entry.values);
} else if (fieldType == SORTED_SET_SINGLETON) {
SortedSetEntry entry = readSortedSetEntry(meta, true);
- sortedSets.put(fieldNumber, entry);
- binaries.put(fieldNumber, entry.values);
+ sortedSets.put(info.name, entry);
+ binaries.put(info.name, entry.values);
} else if (fieldType == SORTED_NUMERIC) {
SortedNumericEntry entry = readSortedNumericEntry(meta, false);
- sortedNumerics.put(fieldNumber, entry);
+ sortedNumerics.put(info.name, entry);
} else if (fieldType == SORTED_NUMERIC_SINGLETON) {
SortedNumericEntry entry = readSortedNumericEntry(meta, true);
- sortedNumerics.put(fieldNumber, entry);
+ sortedNumerics.put(info.name, entry);
} else {
- throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta);
+ throw new CorruptIndexException("invalid entry type: " + fieldType + ", field= " + info.name + ", input=" + meta);
}
fieldNumber = meta.readVInt();
}
+ return numEntries;
}
@Override
@@ -227,35 +234,55 @@ class DirectDocValuesProducer extends Do
}
@Override
+ public synchronized Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ resources.addAll(Accountables.namedAccountables("numeric field", numericInstances));
+ resources.addAll(Accountables.namedAccountables("binary field", binaryInstances));
+ resources.addAll(Accountables.namedAccountables("sorted field", sortedInstances));
+ resources.addAll(Accountables.namedAccountables("sorted set field", sortedSetInstances));
+ resources.addAll(Accountables.namedAccountables("sorted numeric field", sortedNumericInstances));
+ resources.addAll(Accountables.namedAccountables("missing bitset field", docsWithFieldInstances));
+ return Collections.unmodifiableList(resources);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(entries=" + numEntries + ")";
+ }
+
+ @Override
public void checkIntegrity() throws IOException {
CodecUtil.checksumEntireFile(data);
}
@Override
public synchronized NumericDocValues getNumeric(FieldInfo field) throws IOException {
- NumericDocValues instance = numericInstances.get(field.number);
+ NumericRawValues instance = numericInstances.get(field.name);
if (instance == null) {
// Lazy load
- instance = loadNumeric(numerics.get(field.number));
- numericInstances.put(field.number, instance);
+ instance = loadNumeric(numerics.get(field.name));
+ numericInstances.put(field.name, instance);
}
- return instance;
+ return instance.numerics;
}
- private NumericDocValues loadNumeric(NumericEntry entry) throws IOException {
+ private NumericRawValues loadNumeric(NumericEntry entry) throws IOException {
+ NumericRawValues ret = new NumericRawValues();
data.seek(entry.offset + entry.missingBytes);
switch (entry.byteWidth) {
case 1:
{
final byte[] values = new byte[entry.count];
data.readBytes(values, 0, entry.count);
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(values));
- return new NumericDocValues() {
+ ret.bytesUsed = RamUsageEstimator.sizeOf(values);
+ ramBytesUsed.addAndGet(ret.bytesUsed);
+ ret.numerics = new NumericDocValues() {
@Override
public long get(int idx) {
return values[idx];
}
};
+ return ret;
}
case 2:
@@ -264,13 +291,15 @@ class DirectDocValuesProducer extends Do
for(int i=0;i<entry.count;i++) {
values[i] = data.readShort();
}
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(values));
- return new NumericDocValues() {
+ ret.bytesUsed = RamUsageEstimator.sizeOf(values);
+ ramBytesUsed.addAndGet(ret.bytesUsed);
+ ret.numerics = new NumericDocValues() {
@Override
public long get(int idx) {
return values[idx];
}
};
+ return ret;
}
case 4:
@@ -279,13 +308,15 @@ class DirectDocValuesProducer extends Do
for(int i=0;i<entry.count;i++) {
values[i] = data.readInt();
}
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(values));
- return new NumericDocValues() {
+ ret.bytesUsed = RamUsageEstimator.sizeOf(values);
+ ramBytesUsed.addAndGet(ret.bytesUsed);
+ ret.numerics = new NumericDocValues() {
@Override
public long get(int idx) {
return values[idx];
}
};
+ return ret;
}
case 8:
@@ -294,13 +325,15 @@ class DirectDocValuesProducer extends Do
for(int i=0;i<entry.count;i++) {
values[i] = data.readLong();
}
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(values));
- return new NumericDocValues() {
+ ret.bytesUsed = RamUsageEstimator.sizeOf(values);
+ ramBytesUsed.addAndGet(ret.bytesUsed);
+ ret.numerics = new NumericDocValues() {
@Override
public long get(int idx) {
return values[idx];
}
};
+ return ret;
}
default:
@@ -310,11 +343,11 @@ class DirectDocValuesProducer extends Do
@Override
public synchronized BinaryDocValues getBinary(FieldInfo field) throws IOException {
- BinaryRawValues instance = binaryInstances.get(field.number);
+ BinaryRawValues instance = binaryInstances.get(field.name);
if (instance == null) {
// Lazy load
- instance = loadBinary(binaries.get(field.number));
- binaryInstances.put(field.number, instance);
+ instance = loadBinary(binaries.get(field.name));
+ binaryInstances.put(field.name, instance);
}
final byte[] bytes = instance.bytes;
final int[] address = instance.address;
@@ -354,17 +387,17 @@ class DirectDocValuesProducer extends Do
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
- final SortedEntry entry = sorteds.get(field.number);
+ final SortedEntry entry = sorteds.get(field.name);
SortedRawValues instance;
synchronized (this) {
- instance = sortedInstances.get(field.number);
+ instance = sortedInstances.get(field.name);
if (instance == null) {
// Lazy load
instance = loadSorted(field);
- sortedInstances.put(field.number, instance);
+ sortedInstances.put(field.name, instance);
}
}
- return newSortedInstance(instance.docToOrd, getBinary(field), entry.values.count);
+ return newSortedInstance(instance.docToOrd.numerics, getBinary(field), entry.values.count);
}
private SortedDocValues newSortedInstance(final NumericDocValues docToOrd, final BinaryDocValues values, final int count) {
@@ -392,8 +425,8 @@ class DirectDocValuesProducer extends Do
}
private SortedRawValues loadSorted(FieldInfo field) throws IOException {
- final SortedEntry entry = sorteds.get(field.number);
- final NumericDocValues docToOrd = loadNumeric(entry.docToOrd);
+ final SortedEntry entry = sorteds.get(field.name);
+ final NumericRawValues docToOrd = loadNumeric(entry.docToOrd);
final SortedRawValues values = new SortedRawValues();
values.docToOrd = docToOrd;
return values;
@@ -401,21 +434,21 @@ class DirectDocValuesProducer extends Do
@Override
public synchronized SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
- SortedNumericRawValues instance = sortedNumericInstances.get(field.number);
- final SortedNumericEntry entry = sortedNumerics.get(field.number);
+ SortedNumericRawValues instance = sortedNumericInstances.get(field.name);
+ final SortedNumericEntry entry = sortedNumerics.get(field.name);
if (instance == null) {
// Lazy load
instance = loadSortedNumeric(entry);
- sortedNumericInstances.put(field.number, instance);
+ sortedNumericInstances.put(field.name, instance);
}
if (entry.docToAddress == null) {
- final NumericDocValues single = instance.values;
- final Bits docsWithField = getMissingBits(field.number, entry.values.missingOffset, entry.values.missingBytes);
+ final NumericDocValues single = instance.values.numerics;
+ final Bits docsWithField = getMissingBits(field, entry.values.missingOffset, entry.values.missingBytes);
return DocValues.singleton(single, docsWithField);
} else {
- final NumericDocValues docToAddress = instance.docToAddress;
- final NumericDocValues values = instance.values;
+ final NumericDocValues docToAddress = instance.docToAddress.numerics;
+ final NumericDocValues values = instance.values.numerics;
return new SortedNumericDocValues() {
int valueStart;
@@ -451,20 +484,20 @@ class DirectDocValuesProducer extends Do
@Override
public synchronized SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
- SortedSetRawValues instance = sortedSetInstances.get(field.number);
- final SortedSetEntry entry = sortedSets.get(field.number);
+ SortedSetRawValues instance = sortedSetInstances.get(field.name);
+ final SortedSetEntry entry = sortedSets.get(field.name);
if (instance == null) {
// Lazy load
instance = loadSortedSet(entry);
- sortedSetInstances.put(field.number, instance);
+ sortedSetInstances.put(field.name, instance);
}
if (instance.docToOrdAddress == null) {
- SortedDocValues sorted = newSortedInstance(instance.ords, getBinary(field), entry.values.count);
+ SortedDocValues sorted = newSortedInstance(instance.ords.numerics, getBinary(field), entry.values.count);
return DocValues.singleton(sorted);
} else {
- final NumericDocValues docToOrdAddress = instance.docToOrdAddress;
- final NumericDocValues ords = instance.ords;
+ final NumericDocValues docToOrdAddress = instance.docToOrdAddress.numerics;
+ final NumericDocValues ords = instance.ords.numerics;
final BinaryDocValues values = getBinary(field);
// Must make a new instance since the iterator has state:
@@ -524,13 +557,13 @@ class DirectDocValuesProducer extends Do
return instance;
}
- private Bits getMissingBits(int fieldNumber, final long offset, final long length) throws IOException {
+ private Bits getMissingBits(FieldInfo field, final long offset, final long length) throws IOException {
if (offset == -1) {
return new Bits.MatchAllBits(maxDoc);
} else {
- Bits instance;
+ FixedBitSet instance;
synchronized(this) {
- instance = docsWithFieldInstances.get(fieldNumber);
+ instance = docsWithFieldInstances.get(field.name);
if (instance == null) {
IndexInput data = this.data.clone();
data.seek(offset);
@@ -540,7 +573,7 @@ class DirectDocValuesProducer extends Do
bits[i] = data.readLong();
}
instance = new FixedBitSet(bits, maxDoc);
- docsWithFieldInstances.put(fieldNumber, instance);
+ docsWithFieldInstances.put(field.name, instance);
}
}
return instance;
@@ -557,11 +590,11 @@ class DirectDocValuesProducer extends Do
case SORTED:
return DocValues.docsWithValue(getSorted(field), maxDoc);
case BINARY:
- BinaryEntry be = binaries.get(field.number);
- return getMissingBits(field.number, be.missingOffset, be.missingBytes);
+ BinaryEntry be = binaries.get(field.name);
+ return getMissingBits(field, be.missingOffset, be.missingBytes);
case NUMERIC:
- NumericEntry ne = numerics.get(field.number);
- return getMissingBits(field.number, ne.missingOffset, ne.missingBytes);
+ NumericEntry ne = numerics.get(field.name);
+ return getMissingBits(field, ne.missingOffset, ne.missingBytes);
default:
throw new AssertionError();
}
@@ -572,23 +605,130 @@ class DirectDocValuesProducer extends Do
data.close();
}
- static class BinaryRawValues {
+ static class BinaryRawValues implements Accountable {
byte[] bytes;
int[] address;
+
+ @Override
+ public long ramBytesUsed() {
+ long bytesUsed = RamUsageEstimator.sizeOf(bytes);
+ if (address != null) {
+ bytesUsed += RamUsageEstimator.sizeOf(address);
+ }
+ return bytesUsed;
+ }
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ if (address != null) {
+ resources.add(Accountables.namedAccountable("addresses", RamUsageEstimator.sizeOf(address)));
+ }
+ resources.add(Accountables.namedAccountable("bytes", RamUsageEstimator.sizeOf(bytes)));
+ return Collections.unmodifiableList(resources);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
+ }
+
+ static class NumericRawValues implements Accountable {
+ NumericDocValues numerics;
+ long bytesUsed;
+
+ @Override
+ public long ramBytesUsed() {
+ return bytesUsed;
+ }
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
}
- static class SortedRawValues {
- NumericDocValues docToOrd;
+ static class SortedRawValues implements Accountable {
+ NumericRawValues docToOrd;
+
+ @Override
+ public long ramBytesUsed() {
+ return docToOrd.ramBytesUsed();
+ }
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return docToOrd.getChildResources();
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
}
- static class SortedNumericRawValues {
- NumericDocValues docToAddress;
- NumericDocValues values;
+ static class SortedNumericRawValues implements Accountable {
+ NumericRawValues docToAddress;
+ NumericRawValues values;
+
+ @Override
+ public long ramBytesUsed() {
+ long bytesUsed = values.ramBytesUsed();
+ if (docToAddress != null) {
+ bytesUsed += docToAddress.ramBytesUsed();
+ }
+ return bytesUsed;
+ }
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ if (docToAddress != null) {
+ resources.add(Accountables.namedAccountable("addresses", docToAddress));
+ }
+ resources.add(Accountables.namedAccountable("values", values));
+ return Collections.unmodifiableList(resources);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
}
- static class SortedSetRawValues {
- NumericDocValues docToOrdAddress;
- NumericDocValues ords;
+ static class SortedSetRawValues implements Accountable {
+ NumericRawValues docToOrdAddress;
+ NumericRawValues ords;
+
+ @Override
+ public long ramBytesUsed() {
+ long bytesUsed = ords.ramBytesUsed();
+ if (docToOrdAddress != null) {
+ bytesUsed += docToOrdAddress.ramBytesUsed();
+ }
+ return bytesUsed;
+ }
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ if (docToOrdAddress != null) {
+ resources.add(Accountables.namedAccountable("addresses", docToOrdAddress));
+ }
+ resources.add(Accountables.namedAccountable("ordinals", ords));
+ return Collections.unmodifiableList(resources);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
}
static class NumericEntry {
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java Tue Sep 16 19:00:49 2014
@@ -42,6 +42,7 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -169,12 +170,22 @@ public final class DirectPostingsFormat
}
return sizeInBytes;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Accountables.namedAccountables("field", fields);
+ }
@Override
public void checkIntegrity() throws IOException {
// if we read entirely into ram, we already validated.
// otherwise returned the raw postings reader
}
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(fields=" + fields.size() + ")";
+ }
}
private final static class DirectField extends Terms implements Accountable {
@@ -207,6 +218,11 @@ public final class DirectPostingsFormat
((postings!=null) ? RamUsageEstimator.sizeOf(postings) : 0) +
((payloads!=null) ? RamUsageEstimator.sizeOf(payloads) : 0);
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
}
// TODO: maybe specialize into prx/no-prx/no-frq cases?
@@ -255,6 +271,11 @@ public final class DirectPostingsFormat
return sizeInBytes;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
}
private final byte[] termBytes;
@@ -532,6 +553,16 @@ public final class DirectPostingsFormat
return sizeInBytes;
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public String toString() {
+ return "DirectTerms(terms=" + terms.length + ",postings=" + sumDocFreq + ",positions=" + sumTotalTermFreq + ",docs=" + docCount + ")";
+ }
// Compares in unicode (UTF8) order:
int compare(int ord, BytesRef other) {
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java Tue Sep 16 19:00:49 2014
@@ -24,6 +24,7 @@ import java.util.BitSet;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
+import java.util.List;
import java.util.TreeMap;
import org.apache.lucene.codecs.BlockTermState;
@@ -47,6 +48,8 @@ import org.apache.lucene.store.ByteArray
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -184,7 +187,7 @@ public class FSTOrdTermsReader extends F
}
}
- final class TermsReader extends Terms {
+ final class TermsReader extends Terms implements Accountable {
final FieldInfo fieldInfo;
final long numTerms;
final long sumTotalTermFreq;
@@ -284,6 +287,33 @@ public class FSTOrdTermsReader extends F
return new IntersectTermsEnum(compiled, startTerm);
}
+ @Override
+ public long ramBytesUsed() {
+ long ramBytesUsed = 0;
+ if (index != null) {
+ ramBytesUsed += index.ramBytesUsed();
+ ramBytesUsed += RamUsageEstimator.sizeOf(metaBytesBlock);
+ ramBytesUsed += RamUsageEstimator.sizeOf(metaLongsBlock);
+ ramBytesUsed += RamUsageEstimator.sizeOf(skipInfo);
+ ramBytesUsed += RamUsageEstimator.sizeOf(statsBlock);
+ }
+ return ramBytesUsed;
+ }
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ if (index == null) {
+ return Collections.emptyList();
+ } else {
+ return Collections.singletonList(Accountables.namedAccountable("terms", index));
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "FSTOrdTerms(terms=" + numTerms + ",postings=" + sumDocFreq + ",positions=" + sumTotalTermFreq + ",docs=" + docCount + ")";
+ }
+
// Only wraps common operations for PBF interact
abstract class BaseTermsEnum extends TermsEnum {
@@ -847,18 +877,25 @@ public class FSTOrdTermsReader extends F
public long ramBytesUsed() {
long ramBytesUsed = postingsReader.ramBytesUsed();
for (TermsReader r : fields.values()) {
- if (r.index != null) {
- ramBytesUsed += r.index.ramBytesUsed();
- ramBytesUsed += RamUsageEstimator.sizeOf(r.metaBytesBlock);
- ramBytesUsed += RamUsageEstimator.sizeOf(r.metaLongsBlock);
- ramBytesUsed += RamUsageEstimator.sizeOf(r.skipInfo);
- ramBytesUsed += RamUsageEstimator.sizeOf(r.statsBlock);
- }
+ ramBytesUsed += r.ramBytesUsed();
}
return ramBytesUsed;
}
@Override
+ public Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ resources.addAll(Accountables.namedAccountables("field", fields));
+ resources.add(Accountables.namedAccountable("delegate", postingsReader));
+ return Collections.unmodifiableList(resources);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(fields=" + fields.size() + ",delegate=" + postingsReader + ")";
+ }
+
+ @Override
public void checkIntegrity() throws IOException {
postingsReader.checkIntegrity();
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java Tue Sep 16 19:00:49 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs.memory;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfo.IndexOptions;
@@ -79,6 +80,11 @@ class FSTTermOutputs extends Outputs<FST
return ramBytesUsed;
}
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+
// NOTE: actually, FST nodes are seldom
// identical when outputs on their arcs
// aren't NO_OUTPUTs.
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java Tue Sep 16 19:00:49 2014
@@ -23,6 +23,7 @@ import java.util.BitSet;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
+import java.util.List;
import java.util.TreeMap;
import org.apache.lucene.codecs.BlockTermState;
@@ -45,6 +46,7 @@ import org.apache.lucene.store.ByteArray
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -196,7 +198,25 @@ public class FSTTermsReader extends Fiel
@Override
public long ramBytesUsed() {
- return BASE_RAM_BYTES_USED + dict.ramBytesUsed();
+ long bytesUsed = BASE_RAM_BYTES_USED;
+ if (dict != null) {
+ bytesUsed += dict.ramBytesUsed();
+ }
+ return bytesUsed;
+ }
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ if (dict == null) {
+ return Collections.emptyList();
+ } else {
+ return Collections.singletonList(Accountables.namedAccountable("terms", dict));
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "FSTTerms(terms=" + numTerms + ",postings=" + sumDocFreq + ",positions=" + sumTotalTermFreq + ",docs=" + docCount + ")";
}
@Override
@@ -768,6 +788,19 @@ public class FSTTermsReader extends Fiel
}
@Override
+ public Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ resources.addAll(Accountables.namedAccountables("field", fields));
+ resources.add(Accountables.namedAccountable("delegate", postingsReader));
+ return Collections.unmodifiableCollection(resources);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(fields=" + fields.size() + ",delegate=" + postingsReader + ")";
+ }
+
+ @Override
public void checkIntegrity() throws IOException {
postingsReader.checkIntegrity();
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java Tue Sep 16 19:00:49 2014
@@ -19,7 +19,10 @@ package org.apache.lucene.codecs.memory;
import java.io.IOException;
import java.util.Comparator;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
@@ -42,6 +45,8 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -68,23 +73,23 @@ import org.apache.lucene.util.packed.Pac
*/
class MemoryDocValuesProducer extends DocValuesProducer {
// metadata maps (just file pointers and minimal stuff)
- private final Map<Integer,NumericEntry> numerics = new HashMap<>();
- private final Map<Integer,BinaryEntry> binaries = new HashMap<>();
- private final Map<Integer,FSTEntry> fsts = new HashMap<>();
- private final Map<Integer,SortedSetEntry> sortedSets = new HashMap<>();
- private final Map<Integer,SortedNumericEntry> sortedNumerics = new HashMap<>();
+ private final Map<String,NumericEntry> numerics = new HashMap<>();
+ private final Map<String,BinaryEntry> binaries = new HashMap<>();
+ private final Map<String,FSTEntry> fsts = new HashMap<>();
+ private final Map<String,SortedSetEntry> sortedSets = new HashMap<>();
+ private final Map<String,SortedNumericEntry> sortedNumerics = new HashMap<>();
private final IndexInput data;
// ram instances we have already loaded
- private final Map<Integer,NumericDocValues> numericInstances =
- new HashMap<>();
- private final Map<Integer,BytesAndAddresses> pagedBytesInstances =
- new HashMap<>();
- private final Map<Integer,FST<Long>> fstInstances =
- new HashMap<>();
- private final Map<Integer,Bits> docsWithFieldInstances = new HashMap<>();
- private final Map<Integer,MonotonicBlockPackedReader> addresses = new HashMap<>();
+ private final Map<String,NumericDocValues> numericInstances = new HashMap<>();
+ private final Map<String,BytesAndAddresses> pagedBytesInstances = new HashMap<>();
+ private final Map<String,FST<Long>> fstInstances = new HashMap<>();
+ private final Map<String,FixedBitSet> docsWithFieldInstances = new HashMap<>();
+ private final Map<String,MonotonicBlockPackedReader> addresses = new HashMap<>();
+ private final Map<String,Accountable> numericInfo = new HashMap<>();
+
+ private final int numEntries;
private final int maxDoc;
private final AtomicLong ramBytesUsed;
private final int version;
@@ -117,7 +122,7 @@ class MemoryDocValuesProducer extends Do
version = CodecUtil.checkHeader(in, metaCodec,
VERSION_START,
VERSION_CURRENT);
- readFields(in, state.fieldInfos);
+ numEntries = readFields(in, state.fieldInfos);
CodecUtil.checkFooter(in);
ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
success = true;
@@ -204,24 +209,30 @@ class MemoryDocValuesProducer extends Do
return entry;
}
- private void readFields(IndexInput meta, FieldInfos infos) throws IOException {
+ private int readFields(IndexInput meta, FieldInfos infos) throws IOException {
+ int numEntries = 0;
int fieldNumber = meta.readVInt();
while (fieldNumber != -1) {
+ numEntries++;
+ FieldInfo info = infos.fieldInfo(fieldNumber);
+ if (info == null) {
+ throw new CorruptIndexException("invalid field number: " + fieldNumber + " (resource=" + meta + ")");
+ }
int fieldType = meta.readByte();
if (fieldType == NUMBER) {
- numerics.put(fieldNumber, readNumericEntry(meta));
+ numerics.put(info.name, readNumericEntry(meta));
} else if (fieldType == BYTES) {
- binaries.put(fieldNumber, readBinaryEntry(meta));
+ binaries.put(info.name, readBinaryEntry(meta));
} else if (fieldType == FST) {
- fsts.put(fieldNumber,readFSTEntry(meta));
+ fsts.put(info.name,readFSTEntry(meta));
} else if (fieldType == SORTED_SET) {
SortedSetEntry entry = new SortedSetEntry();
entry.singleton = false;
- sortedSets.put(fieldNumber, entry);
+ sortedSets.put(info.name, entry);
} else if (fieldType == SORTED_SET_SINGLETON) {
SortedSetEntry entry = new SortedSetEntry();
entry.singleton = true;
- sortedSets.put(fieldNumber, entry);
+ sortedSets.put(info.name, entry);
} else if (fieldType == SORTED_NUMERIC) {
SortedNumericEntry entry = new SortedNumericEntry();
entry.singleton = false;
@@ -229,24 +240,25 @@ class MemoryDocValuesProducer extends Do
entry.blockSize = meta.readVInt();
entry.addressOffset = meta.readLong();
entry.valueCount = meta.readLong();
- sortedNumerics.put(fieldNumber, entry);
+ sortedNumerics.put(info.name, entry);
} else if (fieldType == SORTED_NUMERIC_SINGLETON) {
SortedNumericEntry entry = new SortedNumericEntry();
entry.singleton = true;
- sortedNumerics.put(fieldNumber, entry);
+ sortedNumerics.put(info.name, entry);
} else {
- throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta);
+ throw new CorruptIndexException("invalid entry type: " + fieldType + ", fieldName=" + info.name + ", input=" + meta);
}
fieldNumber = meta.readVInt();
}
+ return numEntries;
}
@Override
public synchronized NumericDocValues getNumeric(FieldInfo field) throws IOException {
- NumericDocValues instance = numericInstances.get(field.number);
+ NumericDocValues instance = numericInstances.get(field.name);
if (instance == null) {
instance = loadNumeric(field);
- numericInstances.put(field.number, instance);
+ numericInstances.put(field.name, instance);
}
return instance;
}
@@ -257,12 +269,28 @@ class MemoryDocValuesProducer extends Do
}
@Override
+ public synchronized Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ resources.addAll(Accountables.namedAccountables("numeric field", numericInfo));
+ resources.addAll(Accountables.namedAccountables("pagedbytes field", pagedBytesInstances));
+ resources.addAll(Accountables.namedAccountables("term dict field", fstInstances));
+ resources.addAll(Accountables.namedAccountables("missing bitset field", docsWithFieldInstances));
+ resources.addAll(Accountables.namedAccountables("addresses field", addresses));
+ return Collections.unmodifiableList(resources);
+ }
+
+ @Override
public void checkIntegrity() throws IOException {
CodecUtil.checksumEntireFile(data);
}
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(entries=" + numEntries + ")";
+ }
+
private NumericDocValues loadNumeric(FieldInfo field) throws IOException {
- NumericEntry entry = numerics.get(field.number);
+ NumericEntry entry = numerics.get(field.name);
data.seek(entry.offset + entry.missingBytes);
switch (entry.format) {
case TABLE_COMPRESSED:
@@ -278,6 +306,7 @@ class MemoryDocValuesProducer extends Do
final int bitsPerValue = data.readVInt();
final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, (int)entry.count, bitsPerValue);
ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed());
+ numericInfo.put(field.name, Accountables.namedAccountable("table compressed", ordsReader));
return new NumericDocValues() {
@Override
public long get(int docID) {
@@ -290,6 +319,7 @@ class MemoryDocValuesProducer extends Do
final int bitsPerValueDelta = data.readVInt();
final PackedInts.Reader deltaReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatIDDelta), entry.packedIntsVersion, (int)entry.count, bitsPerValueDelta);
ramBytesUsed.addAndGet(deltaReader.ramBytesUsed());
+ numericInfo.put(field.name, Accountables.namedAccountable("delta compressed", deltaReader));
return new NumericDocValues() {
@Override
public long get(int docID) {
@@ -300,6 +330,7 @@ class MemoryDocValuesProducer extends Do
final int blockSize = data.readVInt();
final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, entry.count, false);
ramBytesUsed.addAndGet(reader.ramBytesUsed());
+ numericInfo.put(field.name, Accountables.namedAccountable("block compressed", reader));
return reader;
case GCD_COMPRESSED:
final long min = data.readLong();
@@ -308,6 +339,7 @@ class MemoryDocValuesProducer extends Do
final int bitsPerValueGCD = data.readVInt();
final PackedInts.Reader quotientReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatIDGCD), entry.packedIntsVersion, (int)entry.count, bitsPerValueGCD);
ramBytesUsed.addAndGet(quotientReader.ramBytesUsed());
+ numericInfo.put(field.name, Accountables.namedAccountable("gcd compressed", quotientReader));
return new NumericDocValues() {
@Override
public long get(int docID) {
@@ -321,14 +353,14 @@ class MemoryDocValuesProducer extends Do
@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
- BinaryEntry entry = binaries.get(field.number);
+ BinaryEntry entry = binaries.get(field.name);
BytesAndAddresses instance;
synchronized (this) {
- instance = pagedBytesInstances.get(field.number);
+ instance = pagedBytesInstances.get(field.name);
if (instance == null) {
instance = loadBinary(field);
- pagedBytesInstances.put(field.number, instance);
+ pagedBytesInstances.put(field.name, instance);
}
}
final PagedBytes.Reader bytesReader = instance.reader;
@@ -363,7 +395,7 @@ class MemoryDocValuesProducer extends Do
private BytesAndAddresses loadBinary(FieldInfo field) throws IOException {
BytesAndAddresses bytesAndAddresses = new BytesAndAddresses();
- BinaryEntry entry = binaries.get(field.number);
+ BinaryEntry entry = binaries.get(field.name);
data.seek(entry.offset);
PagedBytes bytes = new PagedBytes(16);
bytes.copy(data, entry.numBytes);
@@ -379,18 +411,18 @@ class MemoryDocValuesProducer extends Do
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
- final FSTEntry entry = fsts.get(field.number);
+ final FSTEntry entry = fsts.get(field.name);
if (entry.numOrds == 0) {
return DocValues.emptySorted();
}
FST<Long> instance;
synchronized(this) {
- instance = fstInstances.get(field.number);
+ instance = fstInstances.get(field.name);
if (instance == null) {
data.seek(entry.offset);
instance = new FST<>(data, PositiveIntOutputs.getSingleton());
ramBytesUsed.addAndGet(instance.ramBytesUsed());
- fstInstances.put(field.number, instance);
+ fstInstances.put(field.name, instance);
}
}
final NumericDocValues docToOrd = getNumeric(field);
@@ -453,21 +485,21 @@ class MemoryDocValuesProducer extends Do
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
- SortedNumericEntry entry = sortedNumerics.get(field.number);
+ SortedNumericEntry entry = sortedNumerics.get(field.name);
if (entry.singleton) {
NumericDocValues values = getNumeric(field);
- NumericEntry ne = numerics.get(field.number);
- Bits docsWithField = getMissingBits(field.number, ne.missingOffset, ne.missingBytes);
+ NumericEntry ne = numerics.get(field.name);
+ Bits docsWithField = getMissingBits(field, ne.missingOffset, ne.missingBytes);
return DocValues.singleton(values, docsWithField);
} else {
final NumericDocValues values = getNumeric(field);
final MonotonicBlockPackedReader addr;
synchronized (this) {
- MonotonicBlockPackedReader res = addresses.get(field.number);
+ MonotonicBlockPackedReader res = addresses.get(field.name);
if (res == null) {
data.seek(entry.addressOffset);
res = MonotonicBlockPackedReader.of(data, entry.packedIntsVersion, entry.blockSize, entry.valueCount, false);
- addresses.put(field.number, res);
+ addresses.put(field.name, res);
}
addr = res;
}
@@ -521,23 +553,23 @@ class MemoryDocValuesProducer extends Do
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
- SortedSetEntry sortedSetEntry = sortedSets.get(field.number);
+ SortedSetEntry sortedSetEntry = sortedSets.get(field.name);
if (sortedSetEntry.singleton) {
return DocValues.singleton(getSorted(field));
}
- final FSTEntry entry = fsts.get(field.number);
+ final FSTEntry entry = fsts.get(field.name);
if (entry.numOrds == 0) {
return DocValues.emptySortedSet(); // empty FST!
}
FST<Long> instance;
synchronized(this) {
- instance = fstInstances.get(field.number);
+ instance = fstInstances.get(field.name);
if (instance == null) {
data.seek(entry.offset);
instance = new FST<>(data, PositiveIntOutputs.getSingleton());
ramBytesUsed.addAndGet(instance.ramBytesUsed());
- fstInstances.put(field.number, instance);
+ fstInstances.put(field.name, instance);
}
}
final BinaryDocValues docToOrds = getBinary(field);
@@ -612,13 +644,13 @@ class MemoryDocValuesProducer extends Do
};
}
- private Bits getMissingBits(int fieldNumber, final long offset, final long length) throws IOException {
+ private Bits getMissingBits(FieldInfo field, final long offset, final long length) throws IOException {
if (offset == -1) {
return new Bits.MatchAllBits(maxDoc);
} else {
- Bits instance;
+ FixedBitSet instance;
synchronized(this) {
- instance = docsWithFieldInstances.get(fieldNumber);
+ instance = docsWithFieldInstances.get(field.name);
if (instance == null) {
IndexInput data = this.data.clone();
data.seek(offset);
@@ -628,7 +660,7 @@ class MemoryDocValuesProducer extends Do
bits[i] = data.readLong();
}
instance = new FixedBitSet(bits, maxDoc);
- docsWithFieldInstances.put(fieldNumber, instance);
+ docsWithFieldInstances.put(field.name, instance);
}
}
return instance;
@@ -645,11 +677,11 @@ class MemoryDocValuesProducer extends Do
case SORTED:
return DocValues.docsWithValue(getSorted(field), maxDoc);
case BINARY:
- BinaryEntry be = binaries.get(field.number);
- return getMissingBits(field.number, be.missingOffset, be.missingBytes);
+ BinaryEntry be = binaries.get(field.name);
+ return getMissingBits(field, be.missingOffset, be.missingBytes);
case NUMERIC:
- NumericEntry ne = numerics.get(field.number);
- return getMissingBits(field.number, ne.missingOffset, ne.missingBytes);
+ NumericEntry ne = numerics.get(field.name);
+ return getMissingBits(field, ne.missingOffset, ne.missingBytes);
default:
throw new AssertionError();
}
@@ -697,9 +729,28 @@ class MemoryDocValuesProducer extends Do
long valueCount;
}
- static class BytesAndAddresses {
+ static class BytesAndAddresses implements Accountable {
PagedBytes.Reader reader;
MonotonicBlockPackedReader addresses;
+
+ @Override
+ public long ramBytesUsed() {
+ long bytesUsed = reader.ramBytesUsed();
+ if (addresses != null) {
+ bytesUsed += addresses.ramBytesUsed();
+ }
+ return bytesUsed;
+ }
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ List<Accountable> resources = new ArrayList<>();
+ if (addresses != null) {
+ resources.add(Accountables.namedAccountable("addresses", addresses));
+ }
+ resources.add(Accountables.namedAccountable("term bytes", reader));
+ return Collections.unmodifiableList(resources);
+ }
}
// exposes FSTEnum directly as a TermsEnum: avoids binary-search next()
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java Tue Sep 16 19:00:49 2014
@@ -48,6 +48,7 @@ import org.apache.lucene.store.IndexInpu
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -969,6 +970,20 @@ public final class MemoryPostingsFormat
public long ramBytesUsed() {
return ((fst!=null) ? fst.ramBytesUsed() : 0);
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ if (fst == null) {
+ return Collections.emptyList();
+ } else {
+ return Collections.singletonList(Accountables.namedAccountable("terms", fst));
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "MemoryTerms(terms=" + termCount + ",postings=" + sumDocFreq + ",positions=" + sumTotalTermFreq + ",docs=" + docCount + ")";
+ }
}
@Override
@@ -1029,6 +1044,16 @@ public final class MemoryPostingsFormat
}
@Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Accountables.namedAccountables("field", fields);
+ }
+
+ @Override
+ public String toString() {
+ return "MemoryPostings(fields=" + fields.size() + ")";
+ }
+
+ @Override
public void checkIntegrity() throws IOException {}
};
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java Tue Sep 16 19:00:49 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.pulsing
*/
import java.io.IOException;
+import java.util.Collections;
import java.util.IdentityHashMap;
import java.util.Map;
import java.util.TreeMap;
@@ -35,6 +36,7 @@ import org.apache.lucene.index.TermState
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeImpl;
@@ -658,4 +660,9 @@ public class PulsingPostingsReader exten
public void checkIntegrity() throws IOException {
wrappedPostingsReader.checkIntegrity();
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java Tue Sep 16 19:00:49 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.sep;
*/
import java.io.IOException;
+import java.util.Collections;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
@@ -34,6 +35,7 @@ import org.apache.lucene.store.DataInput
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -711,4 +713,9 @@ public class SepPostingsReader extends P
public void checkIntegrity() throws IOException {
// TODO: remove sep layout, its fallen behind on features...
}
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java Tue Sep 16 19:00:49 2014
@@ -34,6 +34,7 @@ import java.nio.charset.StandardCharsets
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.ParseException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
@@ -53,6 +54,7 @@ import org.apache.lucene.index.SortedSet
import org.apache.lucene.store.BufferedChecksumIndexInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -509,6 +511,16 @@ class SimpleTextDocValuesReader extends
}
@Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(fields=" + fields.size() + ")";
+ }
+
+ @Override
public void checkIntegrity() throws IOException {
BytesRefBuilder scratch = new BytesRefBuilder();
IndexInput clone = data.clone();
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java?rev=1625356&r1=1625355&r2=1625356&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java Tue Sep 16 19:00:49 2014
@@ -49,6 +49,7 @@ import org.apache.lucene.store.BufferedC
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -602,6 +603,20 @@ class SimpleTextFieldsReader extends Fie
}
@Override
+ public Iterable<? extends Accountable> getChildResources() {
+ if (fst == null) {
+ return Collections.emptyList();
+ } else {
+ return Collections.singletonList(Accountables.namedAccountable("term cache", fst));
+ }
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(terms=" + termCount + ",postings=" + sumDocFreq + ",positions=" + sumTotalTermFreq + ",docs=" + docCount + ")";
+ }
+
+ @Override
public TermsEnum iterator(TermsEnum reuse) throws IOException {
if (fst != null) {
return new SimpleTextTermsEnum(fst, fieldInfo.getIndexOptions());
@@ -698,5 +713,15 @@ class SimpleTextFieldsReader extends Fie
}
@Override
+ public synchronized Iterable<? extends Accountable> getChildResources() {
+ return Accountables.namedAccountables("field", termsCache);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(fields=" + fields.size() + ")";
+ }
+
+ @Override
public void checkIntegrity() throws IOException {}
}