You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/06/18 17:46:00 UTC
svn commit: r1603492 [1/2] - in /lucene/dev/branches/lucene5752: ./ lucene/
lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/
lucene/codecs/src/java/org/apache/lucene/codecs/bloom/
lucene/codecs/src/java/org/apache/lucene/codec...
Author: mikemccand
Date: Wed Jun 18 15:45:58 2014
New Revision: 1603492
URL: http://svn.apache.org/r1603492
Log:
LUCENE-5752: merge trunk
Added:
lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java
- copied unchanged from r1603490, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java
lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaConcurrent.java
- copied unchanged from r1603490, lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaConcurrent.java
Removed:
lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaAddField.java
lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaAddFields.java
lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaCopyFields.java
Modified:
lucene/dev/branches/lucene5752/ (props changed)
lucene/dev/branches/lucene5752/lucene/ (props changed)
lucene/dev/branches/lucene5752/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene5752/lucene/codecs/ (props changed)
lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java
lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java
lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java
lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/blockterms/TestFixedGapPostingsFormat.java
lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestDirectPostingsFormat.java
lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestMemoryPostingsFormat.java
lucene/dev/branches/lucene5752/lucene/core/ (props changed)
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/Version.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java
lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java
lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java
lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java
lucene/dev/branches/lucene5752/lucene/misc/ (props changed)
lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java
lucene/dev/branches/lucene5752/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java
lucene/dev/branches/lucene5752/lucene/spatial/ (props changed)
lucene/dev/branches/lucene5752/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
lucene/dev/branches/lucene5752/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
lucene/dev/branches/lucene5752/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
lucene/dev/branches/lucene5752/solr/ (props changed)
lucene/dev/branches/lucene5752/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene5752/solr/contrib/ (props changed)
lucene/dev/branches/lucene5752/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java
lucene/dev/branches/lucene5752/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
lucene/dev/branches/lucene5752/solr/core/ (props changed)
lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/Overseer.java
lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/cloud/ZkController.java
lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
lucene/dev/branches/lucene5752/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
lucene/dev/branches/lucene5752/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java
Modified: lucene/dev/branches/lucene5752/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/CHANGES.txt?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5752/lucene/CHANGES.txt Wed Jun 18 15:45:58 2014
@@ -94,6 +94,10 @@ Other
* LUCENE-5563: Removed sep layout: which has fallen behind on features and doesn't
perform as well as other options. (Robert Muir)
+======================= Lucene 4.10.0 ======================
+
+(No Changes)
+
======================= Lucene 4.9.0 =======================
Changes in Runtime Behavior
Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java Wed Jun 18 15:45:58 2014
@@ -59,8 +59,7 @@ public class FixedGapTermsIndexReader ex
private final static int PAGED_BYTES_BITS = 15;
// all fields share this single logical byte[]
- private final PagedBytes termBytes = new PagedBytes(PAGED_BYTES_BITS);
- private PagedBytes.Reader termBytesReader;
+ private final PagedBytes.Reader termBytesReader;
final HashMap<FieldInfo,FieldIndexData> fields = new HashMap<>();
@@ -71,6 +70,7 @@ public class FixedGapTermsIndexReader ex
public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, Comparator<BytesRef> termComp, String segmentSuffix, IOContext context)
throws IOException {
+ final PagedBytes termBytes = new PagedBytes(PAGED_BYTES_BITS);
this.termComp = termComp;
@@ -115,7 +115,7 @@ public class FixedGapTermsIndexReader ex
throw new CorruptIndexException("invalid packedIndexStart: " + packedIndexStart + " indexStart: " + indexStart + "numIndexTerms: " + numIndexTerms + " (resource=" + in + ")");
}
final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
- FieldIndexData previous = fields.put(fieldInfo, new FieldIndexData(in, indexStart, termsStart, packedIndexStart, packedOffsetsStart, numIndexTerms));
+ FieldIndexData previous = fields.put(fieldInfo, new FieldIndexData(in, termBytes, indexStart, termsStart, packedIndexStart, packedOffsetsStart, numIndexTerms));
if (previous != null) {
throw new CorruptIndexException("duplicate field: " + fieldInfo.name + " (resource=" + in + ")");
}
@@ -238,7 +238,7 @@ public class FixedGapTermsIndexReader ex
final long numIndexTerms;
final long termsStart;
- public FieldIndexData(IndexInput in, long indexStart, long termsStart, long packedIndexStart, long packedOffsetsStart, long numIndexTerms) throws IOException {
+ public FieldIndexData(IndexInput in, PagedBytes termBytes, long indexStart, long termsStart, long packedIndexStart, long packedOffsetsStart, long numIndexTerms) throws IOException {
this.termsStart = termsStart;
termBytesStart = termBytes.getPointer();
@@ -292,9 +292,7 @@ public class FixedGapTermsIndexReader ex
@Override
public long ramBytesUsed() {
- long sizeInBytes = ((termBytes!=null) ? termBytes.ramBytesUsed() : 0) +
- ((termBytesReader!=null)? termBytesReader.ramBytesUsed() : 0);
-
+ long sizeInBytes = ((termBytesReader!=null)? termBytesReader.ramBytesUsed() : 0);
for(FieldIndexData entry : fields.values()) {
sizeInBytes += entry.ramBytesUsed();
}
Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java Wed Jun 18 15:45:58 2014
@@ -151,7 +151,7 @@ public final class BloomFilteringPosting
return new BloomFilteredFieldsProducer(state);
}
- public class BloomFilteredFieldsProducer extends FieldsProducer {
+ static class BloomFilteredFieldsProducer extends FieldsProducer {
private FieldsProducer delegateFieldsProducer;
HashMap<String,FuzzySet> bloomsByFieldName = new HashMap<>();
Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java Wed Jun 18 15:45:58 2014
@@ -169,11 +169,16 @@ public final class DirectPostingsFormat
private final static class DirectField extends Terms implements Accountable {
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DirectField.class);
+
private static abstract class TermAndSkip implements Accountable {
public int[] skips;
}
private static final class LowFreqTerm extends TermAndSkip {
+
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(HighFreqTerm.class);
+
public final int[] postings;
public final byte[] payloads;
public final int docFreq;
@@ -188,13 +193,17 @@ public final class DirectPostingsFormat
@Override
public long ramBytesUsed() {
- return ((postings!=null) ? RamUsageEstimator.sizeOf(postings) : 0) +
+ return BASE_RAM_BYTES_USED +
+ ((postings!=null) ? RamUsageEstimator.sizeOf(postings) : 0) +
((payloads!=null) ? RamUsageEstimator.sizeOf(payloads) : 0);
}
}
// TODO: maybe specialize into prx/no-prx/no-frq cases?
private static final class HighFreqTerm extends TermAndSkip {
+
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(HighFreqTerm.class);
+
public final long totalTermFreq;
public final int[] docIDs;
public final int[] freqs;
@@ -211,19 +220,22 @@ public final class DirectPostingsFormat
@Override
public long ramBytesUsed() {
- long sizeInBytes = 0;
+ long sizeInBytes = BASE_RAM_BYTES_USED;
sizeInBytes += (docIDs!=null)? RamUsageEstimator.sizeOf(docIDs) : 0;
sizeInBytes += (freqs!=null)? RamUsageEstimator.sizeOf(freqs) : 0;
if(positions != null) {
+ sizeInBytes += RamUsageEstimator.shallowSizeOf(positions);
for(int[] position : positions) {
sizeInBytes += (position!=null) ? RamUsageEstimator.sizeOf(position) : 0;
}
}
if (payloads != null) {
+ sizeInBytes += RamUsageEstimator.shallowSizeOf(payloads);
for(byte[][] payload : payloads) {
if(payload != null) {
+ sizeInBytes += RamUsageEstimator.shallowSizeOf(payload);
for(byte[] pload : payload) {
sizeInBytes += (pload!=null) ? RamUsageEstimator.sizeOf(pload) : 0;
}
@@ -494,7 +506,7 @@ public final class DirectPostingsFormat
@Override
public long ramBytesUsed() {
- long sizeInBytes = 0;
+ long sizeInBytes = BASE_RAM_BYTES_USED;
sizeInBytes += ((termBytes!=null) ? RamUsageEstimator.sizeOf(termBytes) : 0);
sizeInBytes += ((termOffsets!=null) ? RamUsageEstimator.sizeOf(termOffsets) : 0);
sizeInBytes += ((skips!=null) ? RamUsageEstimator.sizeOf(skips) : 0);
@@ -502,6 +514,7 @@ public final class DirectPostingsFormat
sizeInBytes += ((sameCounts!=null) ? RamUsageEstimator.sizeOf(sameCounts) : 0);
if(terms!=null) {
+ sizeInBytes += RamUsageEstimator.shallowSizeOf(terms);
for(TermAndSkip termAndSkip : terms) {
sizeInBytes += (termAndSkip!=null) ? termAndSkip.ramBytesUsed() : 0;
}
Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java Wed Jun 18 15:45:58 2014
@@ -827,7 +827,7 @@ public class FSTOrdTermsReader extends F
@Override
public long ramBytesUsed() {
- long ramBytesUsed = 0;
+ long ramBytesUsed = postingsReader.ramBytesUsed();
for (TermsReader r : fields.values()) {
if (r.index != null) {
ramBytesUsed += r.index.ramBytesUsed();
Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java Wed Jun 18 15:45:58 2014
@@ -24,6 +24,8 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.fst.Outputs;
/**
@@ -46,7 +48,8 @@ class FSTTermOutputs extends Outputs<FST
* On an FST, only long[] part is 'shared' and pushed towards root.
* byte[] and term stats will be kept on deeper arcs.
*/
- static class TermData {
+ static class TermData implements Accountable {
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(TermData.class);
long[] longs;
byte[] bytes;
int docFreq;
@@ -64,6 +67,18 @@ class FSTTermOutputs extends Outputs<FST
this.totalTermFreq = totalTermFreq;
}
+ @Override
+ public long ramBytesUsed() {
+ long ramBytesUsed = BASE_RAM_BYTES_USED;
+ if (longs != null) {
+ ramBytesUsed += RamUsageEstimator.sizeOf(longs);
+ }
+ if (bytes != null) {
+ ramBytesUsed += RamUsageEstimator.sizeOf(bytes);
+ }
+ return ramBytesUsed;
+ }
+
// NOTE: actually, FST nodes are seldom
// identical when outputs on their arcs
// aren't NO_OUTPUTs.
@@ -113,6 +128,11 @@ class FSTTermOutputs extends Outputs<FST
}
@Override
+ public long ramBytesUsed(TermData output) {
+ return output.ramBytesUsed();
+ }
+
+ @Override
//
// The return value will be the smaller one, when these two are
// 'comparable', i.e.
Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java Wed Jun 18 15:45:58 2014
@@ -41,6 +41,7 @@ import org.apache.lucene.store.IndexInpu
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.automaton.ByteRunAutomaton;
import org.apache.lucene.util.automaton.CompiledAutomaton;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -165,7 +166,9 @@ public class FSTTermsReader extends Fiel
}
}
- final class TermsReader extends Terms {
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(TermsReader.class);
+ final class TermsReader extends Terms implements Accountable {
+
final FieldInfo fieldInfo;
final long numTerms;
final long sumTotalTermFreq;
@@ -185,6 +188,11 @@ public class FSTTermsReader extends Fiel
}
@Override
+ public long ramBytesUsed() {
+ return BASE_RAM_BYTES_USED + dict.ramBytesUsed();
+ }
+
+ @Override
public boolean hasFreqs() {
return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
}
@@ -732,9 +740,9 @@ public class FSTTermsReader extends Fiel
@Override
public long ramBytesUsed() {
- long ramBytesUsed = 0;
+ long ramBytesUsed = postingsReader.ramBytesUsed();
for (TermsReader r : fields.values()) {
- ramBytesUsed += r.dict == null ? 0 : r.dict.ramBytesUsed();
+ ramBytesUsed += r.ramBytesUsed();
}
return ramBytesUsed;
}
Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java Wed Jun 18 15:45:58 2014
@@ -55,11 +55,17 @@ import org.apache.lucene.store.ChecksumI
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.StringHelper;
class SimpleTextDocValuesReader extends DocValuesProducer {
+ private static final long BASE_RAM_BYTES_USED =
+ RamUsageEstimator.shallowSizeOfInstance(SimpleTextDocValuesReader.class)
+ + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class);
+
static class OneField {
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(OneField.class);
long dataStartFilePointer;
String pattern;
String ordPattern;
@@ -500,7 +506,8 @@ class SimpleTextDocValuesReader extends
@Override
public long ramBytesUsed() {
- return 0;
+ return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(scratch.bytes)
+ + fields.size() * (RamUsageEstimator.NUM_BYTES_OBJECT_REF * 2L + OneField.BASE_RAM_BYTES_USED);
}
@Override
Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java Wed Jun 18 15:45:58 2014
@@ -55,6 +55,7 @@ import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.fst.Builder;
@@ -65,6 +66,11 @@ import org.apache.lucene.util.fst.Positi
import org.apache.lucene.util.fst.Util;
class SimpleTextFieldsReader extends FieldsProducer {
+
+ private static final long BASE_RAM_BYTES_USED =
+ RamUsageEstimator.shallowSizeOfInstance(SimpleTextFieldsReader.class)
+ + RamUsageEstimator.shallowSizeOfInstance(TreeMap.class);
+
private final TreeMap<String,Long> fields;
private final IndexInput in;
private final FieldInfos fieldInfos;
@@ -497,6 +503,10 @@ class SimpleTextFieldsReader extends Fie
}
}
+ private static final long TERMS_BASE_RAM_BYTES_USED =
+ RamUsageEstimator.shallowSizeOfInstance(SimpleTextTerms.class)
+ + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class)
+ + RamUsageEstimator.shallowSizeOfInstance(CharsRef.class);
private class SimpleTextTerms extends Terms implements Accountable {
private final long termsStart;
private final FieldInfo fieldInfo;
@@ -581,7 +591,8 @@ class SimpleTextFieldsReader extends Fie
@Override
public long ramBytesUsed() {
- return (fst!=null) ? fst.ramBytesUsed() : 0;
+ return TERMS_BASE_RAM_BYTES_USED + (fst!=null ? fst.ramBytesUsed() : 0)
+ + RamUsageEstimator.sizeOf(scratch.bytes) + RamUsageEstimator.sizeOf(scratchUTF16.chars);
}
@Override
@@ -643,14 +654,14 @@ class SimpleTextFieldsReader extends Fie
@Override
synchronized public Terms terms(String field) throws IOException {
- Terms terms = termsCache.get(field);
+ SimpleTextTerms terms = termsCache.get(field);
if (terms == null) {
Long fp = fields.get(field);
if (fp == null) {
return null;
} else {
terms = new SimpleTextTerms(field, fp, maxDoc);
- termsCache.put(field, (SimpleTextTerms) terms);
+ termsCache.put(field, terms);
}
}
return terms;
@@ -667,8 +678,8 @@ class SimpleTextFieldsReader extends Fie
}
@Override
- public long ramBytesUsed() {
- long sizeInBytes = 0;
+ public synchronized long ramBytesUsed() {
+ long sizeInBytes = BASE_RAM_BYTES_USED + fields.size() * 2 * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
for(SimpleTextTerms simpleTextTerms : termsCache.values()) {
sizeInBytes += (simpleTextTerms!=null) ? simpleTextTerms.ramBytesUsed() : 0;
}
Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java Wed Jun 18 15:45:58 2014
@@ -36,6 +36,7 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.UnicodeUtil;
@@ -48,6 +49,12 @@ import static org.apache.lucene.codecs.s
* @lucene.experimental
*/
public class SimpleTextStoredFieldsReader extends StoredFieldsReader {
+
+ private static final long BASE_RAM_BYTES_USED =
+ RamUsageEstimator.shallowSizeOfInstance(SimpleTextStoredFieldsReader.class)
+ + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class)
+ + RamUsageEstimator.shallowSizeOfInstance(CharsRef.class);
+
private long offsets[]; /* docid -> offset in .fld file */
private IndexInput in;
private BytesRef scratch = new BytesRef();
@@ -199,7 +206,8 @@ public class SimpleTextStoredFieldsReade
@Override
public long ramBytesUsed() {
- return 0;
+ return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(offsets)
+ + RamUsageEstimator.sizeOf(scratch.bytes) + RamUsageEstimator.sizeOf(scratchUTF16.chars);
}
@Override
Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java Wed Jun 18 15:45:58 2014
@@ -43,8 +43,10 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.UnicodeUtil;
+
import static org.apache.lucene.codecs.simpletext.SimpleTextTermVectorsWriter.*;
/**
@@ -54,6 +56,12 @@ import static org.apache.lucene.codecs.s
* @lucene.experimental
*/
public class SimpleTextTermVectorsReader extends TermVectorsReader {
+
+ private static final long BASE_RAM_BYTES_USED =
+ RamUsageEstimator.shallowSizeOfInstance(SimpleTextTermVectorsReader.class)
+ + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class)
+ + RamUsageEstimator.shallowSizeOfInstance(CharsRef.class);
+
private long offsets[]; /* docid -> offset in .vec file */
private IndexInput in;
private BytesRef scratch = new BytesRef();
@@ -539,7 +547,7 @@ public class SimpleTextTermVectorsReader
@Override
public long ramBytesUsed() {
- return 0;
+ return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(offsets);
}
@Override
Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/blockterms/TestFixedGapPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/blockterms/TestFixedGapPostingsFormat.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/blockterms/TestFixedGapPostingsFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/blockterms/TestFixedGapPostingsFormat.java Wed Jun 18 15:45:58 2014
@@ -21,7 +21,6 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene41ords.Lucene41WithOrds;
import org.apache.lucene.index.BasePostingsFormatTestCase;
import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.TestUtil;
/**
* Basic tests of a PF using FixedGap terms dictionary
Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestDirectPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestDirectPostingsFormat.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestDirectPostingsFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestDirectPostingsFormat.java Wed Jun 18 15:45:58 2014
@@ -20,7 +20,6 @@ package org.apache.lucene.codecs.memory;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BasePostingsFormatTestCase;
import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.TestUtil;
/**
* Tests DirectPostingsFormat
Modified: lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestMemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestMemoryPostingsFormat.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestMemoryPostingsFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestMemoryPostingsFormat.java Wed Jun 18 15:45:58 2014
@@ -20,7 +20,6 @@ package org.apache.lucene.codecs.memory;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BasePostingsFormatTestCase;
import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.TestUtil;
/**
* Tests MemoryPostingsFormat
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java Wed Jun 18 15:45:58 2014
@@ -27,6 +27,7 @@ import org.apache.lucene.store.ByteArray
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.fst.ByteSequenceOutputs;
import org.apache.lucene.util.fst.FST;
@@ -34,6 +35,11 @@ import org.apache.lucene.util.fst.FST;
/** BlockTree's implementation of {@link Terms}. */
// public for CheckIndex:
public final class FieldReader extends Terms implements Accountable {
+
+ private static final long BASE_RAM_BYTES_USED =
+ RamUsageEstimator.shallowSizeOfInstance(FieldReader.class)
+ + 3 * RamUsageEstimator.shallowSizeOfInstance(BytesRef.class);
+
final long numTerms;
final FieldInfo fieldInfo;
final long sumTotalTermFreq;
@@ -172,6 +178,6 @@ public final class FieldReader extends T
@Override
public long ramBytesUsed() {
- return ((index!=null)? index.ramBytesUsed() : 0);
+ return BASE_RAM_BYTES_USED + ((index!=null)? index.ramBytesUsed() : 0);
}
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java Wed Jun 18 15:45:58 2014
@@ -36,6 +36,8 @@ import org.apache.lucene.util.packed.Pac
*/
public final class CompressingStoredFieldsIndexReader implements Cloneable, Accountable {
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(CompressingStoredFieldsIndexReader.class);
+
final int maxDoc;
final int[] docBases;
final long[] startPointers;
@@ -163,12 +165,14 @@ public final class CompressingStoredFiel
@Override
public long ramBytesUsed() {
- long res = 0;
-
- for(PackedInts.Reader r : docBasesDeltas) {
+ long res = BASE_RAM_BYTES_USED;
+
+ res += RamUsageEstimator.shallowSizeOf(docBasesDeltas);
+ for (PackedInts.Reader r : docBasesDeltas) {
res += r.ramBytesUsed();
}
- for(PackedInts.Reader r : startPointersDeltas) {
+ res += RamUsageEstimator.shallowSizeOf(startPointersDeltas);
+ for (PackedInts.Reader r : startPointersDeltas) {
res += r.ramBytesUsed();
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java Wed Jun 18 15:45:58 2014
@@ -328,7 +328,7 @@ final class Lucene40DocValuesReader exte
final PagedBytes.Reader bytesReader = bytes.freeze(true);
CodecUtil.checkEOF(input);
success = true;
- ramBytesUsed.addAndGet(bytes.ramBytesUsed());
+ ramBytesUsed.addAndGet(bytesReader.ramBytesUsed());
return new BinaryDocValues() {
@Override
@@ -370,7 +370,7 @@ final class Lucene40DocValuesReader exte
CodecUtil.checkEOF(data);
CodecUtil.checkEOF(index);
success = true;
- ramBytesUsed.addAndGet(bytes.ramBytesUsed() + reader.ramBytesUsed());
+ ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + reader.ramBytesUsed());
return new BinaryDocValues() {
@Override
public BytesRef get(int docID) {
@@ -414,7 +414,7 @@ final class Lucene40DocValuesReader exte
final PackedInts.Reader reader = PackedInts.getReader(index);
CodecUtil.checkEOF(data);
CodecUtil.checkEOF(index);
- ramBytesUsed.addAndGet(bytes.ramBytesUsed() + reader.ramBytesUsed());
+ ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + reader.ramBytesUsed());
success = true;
return new BinaryDocValues() {
@Override
@@ -457,7 +457,7 @@ final class Lucene40DocValuesReader exte
final PackedInts.Reader reader = PackedInts.getReader(index);
CodecUtil.checkEOF(data);
CodecUtil.checkEOF(index);
- ramBytesUsed.addAndGet(bytes.ramBytesUsed() + reader.ramBytesUsed());
+ ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + reader.ramBytesUsed());
success = true;
return new BinaryDocValues() {
@@ -540,7 +540,7 @@ final class Lucene40DocValuesReader exte
bytes.copy(data, fixedLength * (long) valueCount);
final PagedBytes.Reader bytesReader = bytes.freeze(true);
final PackedInts.Reader reader = PackedInts.getReader(index);
- ramBytesUsed.addAndGet(bytes.ramBytesUsed() + reader.ramBytesUsed());
+ ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + reader.ramBytesUsed());
return correctBuggyOrds(new SortedDocValues() {
@Override
@@ -578,7 +578,7 @@ final class Lucene40DocValuesReader exte
final PackedInts.Reader ordsReader = PackedInts.getReader(index);
final int valueCount = addressReader.size() - 1;
- ramBytesUsed.addAndGet(bytes.ramBytesUsed() + addressReader.ramBytesUsed() + ordsReader.ramBytesUsed());
+ ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + addressReader.ramBytesUsed() + ordsReader.ramBytesUsed());
return correctBuggyOrds(new SortedDocValues() {
@Override
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java Wed Jun 18 15:45:58 2014
@@ -32,6 +32,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
import java.io.Closeable;
import java.nio.charset.StandardCharsets;
@@ -47,6 +48,9 @@ import static org.apache.lucene.codecs.l
* @lucene.internal
*/
public final class Lucene40StoredFieldsReader extends StoredFieldsReader implements Cloneable, Closeable {
+
+ private static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Lucene40StoredFieldsReader.class);
+
private final FieldInfos fieldInfos;
private final IndexInput fieldsStream;
private final IndexInput indexStream;
@@ -248,7 +252,7 @@ public final class Lucene40StoredFieldsR
@Override
public long ramBytesUsed() {
- return 0;
+ return RAM_BYTES_USED;
}
@Override
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java Wed Jun 18 15:45:58 2014
@@ -43,6 +43,7 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* Concrete class that reads docId(maybe frq,pos,offset,payloads) list
@@ -53,6 +54,8 @@ import org.apache.lucene.util.IOUtils;
*/
public final class Lucene41PostingsReader extends PostingsReaderBase {
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Lucene41PostingsReader.class);
+
private final IndexInput docIn;
private final IndexInput posIn;
private final IndexInput payIn;
@@ -1543,7 +1546,7 @@ public final class Lucene41PostingsReade
@Override
public long ramBytesUsed() {
- return 0;
+ return BASE_RAM_BYTES_USED;
}
@Override
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java Wed Jun 18 15:45:58 2014
@@ -291,7 +291,7 @@ class Lucene42DocValuesProducer extends
final PagedBytes.Reader bytesReader = bytes.freeze(true);
if (entry.minLength == entry.maxLength) {
final int fixedLength = entry.minLength;
- ramBytesUsed.addAndGet(bytes.ramBytesUsed());
+ ramBytesUsed.addAndGet(bytesReader.ramBytesUsed());
return new BinaryDocValues() {
@Override
public BytesRef get(int docID) {
@@ -302,7 +302,7 @@ class Lucene42DocValuesProducer extends
};
} else {
final MonotonicBlockPackedReader addresses = MonotonicBlockPackedReader.of(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
- ramBytesUsed.addAndGet(bytes.ramBytesUsed() + addresses.ramBytesUsed());
+ ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + addresses.ramBytesUsed());
return new BinaryDocValues() {
@Override
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java Wed Jun 18 15:45:58 2014
@@ -199,7 +199,9 @@ public abstract class PerFieldPostingsFo
}
}
- private class FieldsReader extends FieldsProducer {
+ private static class FieldsReader extends FieldsProducer {
+
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FieldsReader.class);
private final Map<String,FieldsProducer> fields = new TreeMap<>();
private final Map<String,FieldsProducer> formats = new HashMap<>();
@@ -258,12 +260,13 @@ public abstract class PerFieldPostingsFo
@Override
public long ramBytesUsed() {
- long sizeInBytes = 0;
+ long ramBytesUsed = BASE_RAM_BYTES_USED;
+ ramBytesUsed += fields.size() * 2L * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+ ramBytesUsed += formats.size() * 2L * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
for(Map.Entry<String,FieldsProducer> entry: formats.entrySet()) {
- sizeInBytes += entry.getKey().length() * RamUsageEstimator.NUM_BYTES_CHAR;
- sizeInBytes += entry.getValue().ramBytesUsed();
+ ramBytesUsed += entry.getValue().ramBytesUsed();
}
- return sizeInBytes;
+ return ramBytesUsed;
}
@Override
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java Wed Jun 18 15:45:58 2014
@@ -39,11 +39,14 @@ import org.apache.lucene.store.IOContext
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.CloseableThreadLocal;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
/** Holds core readers that are shared (unchanged) when
* SegmentReader is cloned or reopened */
final class SegmentCoreReaders implements Accountable {
-
+
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(SegmentCoreReaders.class);
+
// Counts how many other readers share the core objects
// (freqStream, proxStream, tis, etc.) of this reader;
// when coreRef drops to 0, these core objects may be
@@ -213,7 +216,8 @@ final class SegmentCoreReaders implement
@Override
public long ramBytesUsed() {
- return ((normsProducer!=null) ? normsProducer.ramBytesUsed() : 0) +
+ return BASE_RAM_BYTES_USED +
+ ((normsProducer!=null) ? normsProducer.ramBytesUsed() : 0) +
((fields!=null) ? fields.ramBytesUsed() : 0) +
((fieldsReaderOrig!=null)? fieldsReaderOrig.ramBytesUsed() : 0) +
((termVectorsReaderOrig!=null) ? termVectorsReaderOrig.ramBytesUsed() : 0);
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Wed Jun 18 15:45:58 2014
@@ -40,6 +40,7 @@ import org.apache.lucene.util.Accountabl
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.CloseableThreadLocal;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.Version;
/**
@@ -51,6 +52,11 @@ import org.apache.lucene.util.Version;
*/
public final class SegmentReader extends AtomicReader implements Accountable {
+ private static final long BASE_RAM_BYTES_USED =
+ RamUsageEstimator.shallowSizeOfInstance(SegmentReader.class)
+ + RamUsageEstimator.shallowSizeOfInstance(SegmentDocValues.class);
+ private static final long LONG_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Long.class);
+
private final SegmentCommitInfo si;
private final Bits liveDocs;
@@ -594,7 +600,10 @@ public final class SegmentReader extends
@Override
public long ramBytesUsed() {
ensureOpen();
- long ramBytesUsed = 0;
+ long ramBytesUsed = BASE_RAM_BYTES_USED;
+ ramBytesUsed += dvGens.size() * LONG_RAM_BYTES_USED;
+ ramBytesUsed += dvProducers.size() * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+ ramBytesUsed += dvProducersByField.size() * 2 * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
if (dvProducers != null) {
for (DocValuesProducer producer : dvProducers) {
ramBytesUsed += producer.ramBytesUsed();
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java Wed Jun 18 15:45:58 2014
@@ -18,8 +18,7 @@ package org.apache.lucene.util;
*/
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.Arrays;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
@@ -35,9 +34,10 @@ import org.apache.lucene.store.IndexInpu
// TODO: refactor this, byteblockpool, fst.bytestore, and any
// other "shift/mask big arrays". there are too many of these classes!
public final class PagedBytes implements Accountable {
- private final List<byte[]> blocks = new ArrayList<>();
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(PagedBytes.class);
+ private byte[][] blocks = new byte[16][];
+ private int numBlocks;
// TODO: these are unused?
- private final List<Integer> blockEnd = new ArrayList<>();
private final int blockSize;
private final int blockBits;
private final int blockMask;
@@ -54,24 +54,19 @@ public final class PagedBytes implements
*
* @see #freeze */
public final static class Reader implements Accountable {
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Reader.class);
private final byte[][] blocks;
- private final int[] blockEnds;
private final int blockBits;
private final int blockMask;
private final int blockSize;
+ private final long bytesUsedPerBlock;
private Reader(PagedBytes pagedBytes) {
- blocks = new byte[pagedBytes.blocks.size()][];
- for(int i=0;i<blocks.length;i++) {
- blocks[i] = pagedBytes.blocks.get(i);
- }
- blockEnds = new int[blocks.length];
- for(int i=0;i< blockEnds.length;i++) {
- blockEnds[i] = pagedBytes.blockEnd.get(i);
- }
+ blocks = Arrays.copyOf(pagedBytes.blocks, pagedBytes.numBlocks);
blockBits = pagedBytes.blockBits;
blockMask = pagedBytes.blockMask;
blockSize = pagedBytes.blockSize;
+ bytesUsedPerBlock = pagedBytes.bytesUsedPerBlock;
}
/**
@@ -132,7 +127,12 @@ public final class PagedBytes implements
@Override
public long ramBytesUsed() {
- return ((blocks!=null) ? (blockSize * blocks.length) : 0);
+ long size = BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOf(blocks);
+ if (blocks.length > 0) {
+ size += (blocks.length - 1) * bytesUsedPerBlock;
+ size += RamUsageEstimator.sizeOf(blocks[blocks.length - 1]);
+ }
+ return size;
}
}
@@ -144,7 +144,15 @@ public final class PagedBytes implements
this.blockBits = blockBits;
blockMask = blockSize-1;
upto = blockSize;
- bytesUsedPerBlock = blockSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+ bytesUsedPerBlock = RamUsageEstimator.alignObjectSize(blockSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER);
+ numBlocks = 0;
+ }
+
+ private void addBlock(byte[] block) {
+ if (blocks.length == numBlocks) {
+ blocks = Arrays.copyOf(blocks, ArrayUtil.oversize(numBlocks, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
+ }
+ blocks[numBlocks++] = block;
}
/** Read this many bytes from in */
@@ -153,8 +161,7 @@ public final class PagedBytes implements
int left = blockSize - upto;
if (left == 0) {
if (currentBlock != null) {
- blocks.add(currentBlock);
- blockEnd.add(upto);
+ addBlock(currentBlock);
}
currentBlock = new byte[blockSize];
upto = 0;
@@ -179,8 +186,7 @@ public final class PagedBytes implements
int left = blockSize - upto;
if (bytes.length > left || currentBlock==null) {
if (currentBlock != null) {
- blocks.add(currentBlock);
- blockEnd.add(upto);
+ addBlock(currentBlock);
didSkipBytes = true;
}
currentBlock = new byte[blockSize];
@@ -214,8 +220,7 @@ public final class PagedBytes implements
if (currentBlock == null) {
currentBlock = EMPTY_BYTES;
}
- blocks.add(currentBlock);
- blockEnd.add(upto);
+ addBlock(currentBlock);
frozen = true;
currentBlock = null;
return new PagedBytes.Reader(this);
@@ -225,13 +230,18 @@ public final class PagedBytes implements
if (currentBlock == null) {
return 0;
} else {
- return (blocks.size() * ((long) blockSize)) + upto;
+ return (numBlocks * ((long) blockSize)) + upto;
}
}
@Override
public long ramBytesUsed() {
- return (blocks.size() + (currentBlock != null ? 1 : 0)) * bytesUsedPerBlock;
+ long size = BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOf(blocks)
+ + bytesUsedPerBlock * numBlocks;
+ if (currentBlock != null) {
+ size += RamUsageEstimator.sizeOf(currentBlock);
+ }
+ return size;
}
/** Copy bytes in, writing the length as a 1 or 2 byte
@@ -247,8 +257,7 @@ public final class PagedBytes implements
throw new IllegalArgumentException("block size " + blockSize + " is too small to store length " + bytes.length + " bytes");
}
if (currentBlock != null) {
- blocks.add(currentBlock);
- blockEnd.add(upto);
+ addBlock(currentBlock);
}
currentBlock = new byte[blockSize];
upto = 0;
@@ -274,7 +283,7 @@ public final class PagedBytes implements
private byte[] currentBlock;
PagedBytesDataInput() {
- currentBlock = blocks.get(0);
+ currentBlock = blocks[0];
}
@Override
@@ -293,7 +302,7 @@ public final class PagedBytes implements
* {@link #getPosition}. */
public void setPosition(long pos) {
currentBlockIndex = (int) (pos >> blockBits);
- currentBlock = blocks.get(currentBlockIndex);
+ currentBlock = blocks[currentBlockIndex];
currentBlockUpto = (int) (pos & blockMask);
}
@@ -332,7 +341,7 @@ public final class PagedBytes implements
private void nextBlock() {
currentBlockIndex++;
currentBlockUpto = 0;
- currentBlock = blocks.get(currentBlockIndex);
+ currentBlock = blocks[currentBlockIndex];
}
}
@@ -341,8 +350,7 @@ public final class PagedBytes implements
public void writeByte(byte b) {
if (upto == blockSize) {
if (currentBlock != null) {
- blocks.add(currentBlock);
- blockEnd.add(upto);
+ addBlock(currentBlock);
}
currentBlock = new byte[blockSize];
upto = 0;
@@ -359,8 +367,7 @@ public final class PagedBytes implements
if (upto == blockSize) {
if (currentBlock != null) {
- blocks.add(currentBlock);
- blockEnd.add(upto);
+ addBlock(currentBlock);
}
currentBlock = new byte[blockSize];
upto = 0;
@@ -372,8 +379,7 @@ public final class PagedBytes implements
final int blockLeft = blockSize - upto;
if (blockLeft < left) {
System.arraycopy(b, offset, currentBlock, upto, blockLeft);
- blocks.add(currentBlock);
- blockEnd.add(blockSize);
+ addBlock(currentBlock);
currentBlock = new byte[blockSize];
upto = 0;
offset += blockLeft;
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/Version.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/Version.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/Version.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/Version.java Wed Jun 18 15:45:58 2014
@@ -102,6 +102,13 @@ public enum Version {
*/
@Deprecated
LUCENE_4_9,
+
+ /**
+ * Match settings and bugs in Lucene's 4.10 release.
+ * @deprecated (5.0) Use latest
+ */
+ @Deprecated
+ LUCENE_4_10,
/** Match settings and bugs in Lucene's 5.0 release.
* <p>
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java Wed Jun 18 15:45:58 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* An FST {@link Outputs} implementation where each output
@@ -145,4 +146,9 @@ public final class ByteSequenceOutputs e
public String outputToString(BytesRef output) {
return output.toString();
}
+
+ @Override
+ public long ramBytesUsed(BytesRef output) {
+ return super.ramBytesUsed(output) + RamUsageEstimator.sizeOf(output.bytes);
+ }
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java Wed Jun 18 15:45:58 2014
@@ -23,11 +23,17 @@ import java.util.List;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.RamUsageEstimator;
// TODO: merge with PagedBytes, except PagedBytes doesn't
// let you read while writing which FST needs
-class BytesStore extends DataOutput {
+class BytesStore extends DataOutput implements Accountable {
+
+ private static final long BASE_RAM_BYTES_USED =
+ RamUsageEstimator.shallowSizeOfInstance(BytesStore.class)
+ + RamUsageEstimator.shallowSizeOfInstance(ArrayList.class);
private final List<byte[]> blocks = new ArrayList<>();
@@ -465,4 +471,14 @@ class BytesStore extends DataOutput {
}
};
}
+
+ @Override
+ public long ramBytesUsed() {
+ long size = BASE_RAM_BYTES_USED;
+ for (byte[] block : blocks) {
+ size += RamUsageEstimator.sizeOf(block);
+ }
+ return size;
+ }
+
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/FST.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/FST.java Wed Jun 18 15:45:58 2014
@@ -41,6 +41,7 @@ import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.fst.Builder.UnCompiledNode;
import org.apache.lucene.util.packed.GrowableWriter;
import org.apache.lucene.util.packed.PackedInts;
@@ -69,6 +70,10 @@ import org.apache.lucene.util.packed.Pac
* @lucene.experimental
*/
public final class FST<T> implements Accountable {
+
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FST.class);
+ private static final long ARC_SHALLOW_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Arc.class);
+
/** Specifies allowed range of each int input label for
* this FST. */
public static enum INPUT_TYPE {BYTE1, BYTE2, BYTE4};
@@ -390,15 +395,38 @@ public final class FST<T> implements Acc
return inputType;
}
+ private long ramBytesUsed(Arc<T>[] arcs) {
+ long size = 0;
+ if (arcs != null) {
+ size += RamUsageEstimator.shallowSizeOf(arcs);
+ for (Arc<T> arc : arcs) {
+ if (arc != null) {
+ size += ARC_SHALLOW_RAM_BYTES_USED;
+ if (arc.output != null && arc.output != outputs.getNoOutput()) {
+ size += outputs.ramBytesUsed(arc.output);
+ }
+ if (arc.nextFinalOutput != null && arc.nextFinalOutput != outputs.getNoOutput()) {
+ size += outputs.ramBytesUsed(arc.nextFinalOutput);
+ }
+ }
+ }
+ }
+ return size;
+ }
+
@Override
public long ramBytesUsed() {
- long size = bytes.getPosition();
+ long size = BASE_RAM_BYTES_USED;
+ size += bytes.ramBytesUsed();
if (packed) {
size += nodeRefToAddress.ramBytesUsed();
} else if (nodeAddress != null) {
size += nodeAddress.ramBytesUsed();
size += inCounts.ramBytesUsed();
}
+ size += ramBytesUsed(cachedRootArcs);
+ size += ramBytesUsed(assertingCachedRootArcs);
+ size += RamUsageEstimator.sizeOf(bytesPerArc);
return size;
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java Wed Jun 18 15:45:58 2014
@@ -21,6 +21,8 @@ import java.io.IOException;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* Represents the outputs for an FST, providing the basic
@@ -95,4 +97,10 @@ public abstract class Outputs<T> {
public T merge(T first, T second) {
throw new UnsupportedOperationException();
}
+
+ /** Return memory usage for the provided output.
+ * @see Accountable */
+ public long ramBytesUsed(T output) {
+ return RamUsageEstimator.shallowSizeOf(output);
+ }
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java Wed Jun 18 15:45:58 2014
@@ -175,4 +175,16 @@ public class PairOutputs<A,B> extends Ou
public String toString() {
return "PairOutputs<" + outputs1 + "," + outputs2 + ">";
}
+
+ @Override
+ public long ramBytesUsed(Pair<A,B> output) {
+ long ramBytesUsed = super.ramBytesUsed(output);
+ if (output.output1 != null) {
+ ramBytesUsed += outputs1.ramBytesUsed(output.output1);
+ }
+ if (output.output2 != null) {
+ ramBytesUsed += outputs2.ramBytesUsed(output.output2);
+ }
+ return ramBytesUsed;
+ }
}
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java Wed Jun 18 15:45:58 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.lang.reflect.Field;
+import java.util.HashMap;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.lucene49.Lucene49DocValuesFormat;
@@ -37,18 +38,28 @@ import org.apache.lucene.util.TestUtil;
public class TestOrdinalMap extends LuceneTestCase {
- private static final RamUsageTester.Filter ORDINAL_MAP_FILTER = new RamUsageTester.Filter() {
- @Override
- public boolean accept(Field field) {
- if (field.getDeclaringClass().equals(OrdinalMap.class) && field.getName().equals("owner")) {
- return false;
- }
- return true;
+ private static final Field ORDINAL_MAP_OWNER_FIELD;
+ static {
+ try {
+ ORDINAL_MAP_OWNER_FIELD = OrdinalMap.class.getDeclaredField("owner");
+ } catch (Exception e) {
+ throw new Error();
}
+ }
+
+ private static final RamUsageTester.Accumulator ORDINAL_MAP_ACCUMULATOR = new RamUsageTester.Accumulator() {
- public boolean accept(Object o) {
- return o != LongValues.IDENTITY;
+ public long accumulateObject(Object o, long shallowSize, java.util.Map<Field,Object> fieldValues, java.util.Collection<Object> queue) {
+ if (o == LongValues.IDENTITY) {
+ return 0L;
+ }
+ if (o instanceof OrdinalMap) {
+ fieldValues = new HashMap<>(fieldValues);
+ fieldValues.remove(ORDINAL_MAP_OWNER_FIELD);
+ }
+ return super.accumulateObject(o, shallowSize, fieldValues, queue);
}
+
};
public void testRamBytesUsed() throws IOException {
@@ -77,12 +88,12 @@ public class TestOrdinalMap extends Luce
SortedDocValues sdv = ar.getSortedDocValues("sdv");
if (sdv instanceof MultiSortedDocValues) {
OrdinalMap map = ((MultiSortedDocValues) sdv).mapping;
- assertEquals(RamUsageTester.sizeOf(map, ORDINAL_MAP_FILTER), map.ramBytesUsed());
+ assertEquals(RamUsageTester.sizeOf(map, ORDINAL_MAP_ACCUMULATOR), map.ramBytesUsed());
}
SortedSetDocValues ssdv = ar.getSortedSetDocValues("ssdv");
if (ssdv instanceof MultiSortedSetDocValues) {
OrdinalMap map = ((MultiSortedSetDocValues) ssdv).mapping;
- assertEquals(RamUsageTester.sizeOf(map, ORDINAL_MAP_FILTER), map.ramBytesUsed());
+ assertEquals(RamUsageTester.sizeOf(map, ORDINAL_MAP_ACCUMULATOR), map.ramBytesUsed());
}
iw.close();
r.close();
Modified: lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java (original)
+++ lucene/dev/branches/lucene5752/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java Wed Jun 18 15:45:58 2014
@@ -27,6 +27,7 @@ import org.apache.lucene.store.IOContext
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.PagedBytes.Reader;
import org.junit.Ignore;
public class TestPagedBytes extends LuceneTestCase {
@@ -185,4 +186,17 @@ public class TestPagedBytes extends Luce
dir.close();
}
+ public void testRamBytesUsed() {
+ final int blockBits = TestUtil.nextInt(random(), 4, 22);
+ PagedBytes b = new PagedBytes(blockBits);
+ final int totalBytes = random().nextInt(10000);
+ for (long pointer = 0; pointer < totalBytes; ) {
+ BytesRef bytes = new BytesRef(TestUtil.randomSimpleString(random(), 10));
+ pointer = b.copyUsingLengthPrefix(bytes);
+ }
+ assertEquals(RamUsageTester.sizeOf(b), b.ramBytesUsed());
+ final PagedBytes.Reader reader = b.freeze(random().nextBoolean());
+ assertEquals(RamUsageTester.sizeOf(reader), reader.ramBytesUsed());
+ }
+
}
Modified: lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java (original)
+++ lucene/dev/branches/lucene5752/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java Wed Jun 18 15:45:58 2014
@@ -30,10 +30,12 @@ import org.apache.lucene.index.Fields;
import org.apache.lucene.index.FilterAtomicReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.sorter.Sorter.DocMap;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Sort;
import org.apache.lucene.store.IndexInput;
@@ -222,6 +224,32 @@ public class SortingAtomicReader extends
}
}
+ private static class SortingSortedNumericDocValues extends SortedNumericDocValues {
+
+ private final SortedNumericDocValues in;
+ private final Sorter.DocMap docMap;
+
+ SortingSortedNumericDocValues(SortedNumericDocValues in, DocMap docMap) {
+ this.in = in;
+ this.docMap = docMap;
+ }
+
+ @Override
+ public int count() {
+ return in.count();
+ }
+
+ @Override
+ public void setDocument(int doc) {
+ in.setDocument(docMap.newToOld(doc));
+ }
+
+ @Override
+ public long valueAt(int index) {
+ return in.valueAt(index);
+ }
+ }
+
private static class SortingBits implements Bits {
private final Bits in;
@@ -786,6 +814,17 @@ public class SortingAtomicReader extends
if (oldDocValues == null) return null;
return new SortingNumericDocValues(oldDocValues, docMap);
}
+
+ @Override
+ public SortedNumericDocValues getSortedNumericDocValues(String field)
+ throws IOException {
+ final SortedNumericDocValues oldDocValues = in.getSortedNumericDocValues(field);
+ if (oldDocValues == null) {
+ return null;
+ } else {
+ return new SortingSortedNumericDocValues(oldDocValues, docMap);
+ }
+ }
@Override
public SortedDocValues getSortedDocValues(String field) throws IOException {
Modified: lucene/dev/branches/lucene5752/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5752/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java?rev=1603492&r1=1603491&r2=1603492&view=diff
==============================================================================
--- lucene/dev/branches/lucene5752/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java (original)
+++ lucene/dev/branches/lucene5752/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java Wed Jun 18 15:45:58 2014
@@ -36,6 +36,7 @@ import org.apache.lucene.document.Field.
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
@@ -52,6 +53,7 @@ import org.apache.lucene.index.NumericDo
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
@@ -145,6 +147,7 @@ public abstract class SorterTestBase ext
protected static final String DOC_POSITIONS_FIELD = "positions";
protected static final String DOC_POSITIONS_TERM = "$all$";
protected static final String NUMERIC_DV_FIELD = "numeric";
+ protected static final String SORTED_NUMERIC_DV_FIELD = "sorted_numeric";
protected static final String NORMS_FIELD = "norm";
protected static final String BINARY_DV_FIELD = "binary";
protected static final String SORTED_DV_FIELD = "sorted";
@@ -183,6 +186,10 @@ public abstract class SorterTestBase ext
doc.add(new SortedSetDocValuesField(SORTED_SET_DV_FIELD, new BytesRef(Integer.toString(id))));
doc.add(new SortedSetDocValuesField(SORTED_SET_DV_FIELD, new BytesRef(Integer.toString(id + 1))));
}
+ if (defaultCodecSupportsSortedNumeric()) {
+ doc.add(new SortedNumericDocValuesField(SORTED_NUMERIC_DV_FIELD, id));
+ doc.add(new SortedNumericDocValuesField(SORTED_NUMERIC_DV_FIELD, id + 1));
+ }
doc.add(new Field(TERM_VECTORS_FIELD, Integer.toString(id), TERM_VECTORS_TYPE));
return doc;
}
@@ -389,6 +396,20 @@ public abstract class SorterTestBase ext
}
@Test
+ public void testSortedNumericDocValuesField() throws Exception {
+ assumeTrue("default codec does not support SORTED_NUMERIC", defaultCodecSupportsSortedNumeric());
+ SortedNumericDocValues dv = reader.getSortedNumericDocValues(SORTED_NUMERIC_DV_FIELD);
+ int maxDoc = reader.maxDoc();
+ for (int i = 0; i < maxDoc; i++) {
+ dv.setDocument(i);
+ assertEquals(2, dv.count());
+ int value = sortedValues[i].intValue();
+ assertEquals("incorrect sorted-numeric DocValues for doc " + i, value, dv.valueAt(0));
+ assertEquals("incorrect sorted-numeric DocValues for doc " + i, value + 1, dv.valueAt(1));
+ }
+ }
+
+ @Test
public void testTermVectors() throws Exception {
int maxDoc = reader.maxDoc();
for (int i = 0; i < maxDoc; i++) {