You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2014/02/20 16:49:06 UTC
svn commit: r1570240 - in /lucene/dev/branches/branch_4x/lucene: ./
core/src/java/org/apache/lucene/codecs/lucene3x/
Author: jpountz
Date: Thu Feb 20 15:49:05 2014
New Revision: 1570240
URL: http://svn.apache.org/r1570240
Log:
LUCENE-5462: Don't use RamUsageEstimator.sizeOf(Object) in codec producers.
Modified:
lucene/dev/branches/branch_4x/lucene/CHANGES.txt
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1570240&r1=1570239&r2=1570240&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Thu Feb 20 15:49:05 2014
@@ -208,6 +208,10 @@ Bug fixes
* LUCENE-5447: StandardTokenizer should break at consecutive chars matching
Word_Break = MidLetter, MidNum and/or MidNumLet (Steve Rowe)
+
+* LUCENE-5462: RamUsageEstimator.sizeOf(Object) is not used anymore to
+ estimate memory usage of segments. This used to make
+ SegmentReader.ramBytesUsed very CPU-intensive. (Adrien Grand)
API Changes
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java?rev=1570240&r1=1570239&r2=1570240&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java Thu Feb 20 15:49:05 2014
@@ -1077,6 +1077,11 @@ class Lucene3xFields extends FieldsProdu
@Override
public long ramBytesUsed() {
- return RamUsageEstimator.sizeOf(this);
+ if (tis != null) {
+ return tis.ramBytesUsed();
+ } else {
+ // when there is no index, there is almost nothing loaded into RAM
+ return 0L;
+ }
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java?rev=1570240&r1=1570239&r2=1570240&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java Thu Feb 20 15:49:05 2014
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.BinaryDocValues;
@@ -66,6 +67,8 @@ class Lucene3xNormsProducer extends DocV
IndexInput singleNormStream;
final int maxdoc;
+ private final AtomicLong ramBytesUsed;
+
// note: just like segmentreader in 3.x, we open up all the files here (including separate norms) up front.
// but we just don't do any seeks or reading yet.
public Lucene3xNormsProducer(Directory dir, SegmentInfo info, FieldInfos fields, IOContext context) throws IOException {
@@ -125,6 +128,7 @@ class Lucene3xNormsProducer extends DocV
IOUtils.closeWhileHandlingException(openFiles);
}
}
+ ramBytesUsed = new AtomicLong();
}
@Override
@@ -182,6 +186,7 @@ class Lucene3xNormsProducer extends DocV
openFiles.remove(file);
file.close();
}
+ ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes));
instance = new NumericDocValues() {
@Override
public long get(int docID) {
@@ -222,6 +227,6 @@ class Lucene3xNormsProducer extends DocV
@Override
public long ramBytesUsed() {
- return RamUsageEstimator.sizeOf(this);
+ return ramBytesUsed.get();
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java?rev=1570240&r1=1570239&r2=1570240&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java Thu Feb 20 15:49:05 2014
@@ -301,6 +301,7 @@ final class Lucene3xStoredFieldsReader e
@Override
public long ramBytesUsed() {
- return RamUsageEstimator.sizeOf(this);
+ // everything is stored on disk
+ return 0;
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java?rev=1570240&r1=1570239&r2=1570240&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java Thu Feb 20 15:49:05 2014
@@ -740,7 +740,8 @@ class Lucene3xTermVectorsReader extends
@Override
public long ramBytesUsed() {
- return RamUsageEstimator.sizeOf(this);
+ // everything is disk-based
+ return 0;
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReader.java?rev=1570240&r1=1570239&r2=1570240&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReader.java Thu Feb 20 15:49:05 2014
@@ -352,4 +352,8 @@ final class TermInfosReader implements C
get(term, true);
return getThreadResources().termEnum.clone();
}
+
+ long ramBytesUsed() {
+ return index == null ? 0 : index.ramBytesUsed();
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java?rev=1570240&r1=1570239&r2=1570240&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java Thu Feb 20 15:49:05 2014
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.MathUtil;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.PagedBytes.PagedBytesDataInput;
import org.apache.lucene.util.PagedBytes.PagedBytesDataOutput;
import org.apache.lucene.util.PagedBytes;
@@ -49,6 +50,7 @@ class TermInfosReaderIndex {
private final PackedInts.Reader indexToDataOffset;
private final int indexSize;
private final int skipInterval;
+ private final long ramBytesUsed;
/**
* Loads the segment information at segment load time.
@@ -111,6 +113,9 @@ class TermInfosReaderIndex {
dataPagedBytes.freeze(true);
dataInput = dataPagedBytes.getDataInput();
indexToDataOffset = indexToTerms.getMutable();
+
+ ramBytesUsed = fields.length * (RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.shallowSizeOfInstance(Term.class))
+ + dataPagedBytes.ramBytesUsed() + indexToDataOffset.ramBytesUsed();
}
private static int estimatePageBits(long estSize) {
@@ -253,4 +258,9 @@ class TermInfosReaderIndex {
input.setPosition(indexToDataOffset.get(termIndex));
return term.field().compareTo(fields[input.readVInt()].field());
}
+
+ long ramBytesUsed() {
+ return ramBytesUsed;
+ }
+
}