You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/09/04 20:52:20 UTC
svn commit: r1380808 - in /lucene/dev/trunk: ./ lucene/ lucene/core/
lucene/core/src/java/org/apache/lucene/store/
lucene/core/src/test/org/apache/lucene/index/ lucene/test-framework/
lucene/test-framework/src/java/org/apache/lucene/store/ lucene/test-...
Author: mikemccand
Date: Tue Sep 4 18:52:19 2012
New Revision: 1380808
URL: http://svn.apache.org/viewvc?rev=1380808&view=rev
Log:
fix test/test-framework/real bug causing OOME in TestPostingsFormat
Modified:
lucene/dev/trunk/ (props changed)
lucene/dev/trunk/lucene/ (props changed)
lucene/dev/trunk/lucene/CHANGES.txt (contents, props changed)
lucene/dev/trunk/lucene/core/ (props changed)
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
lucene/dev/trunk/lucene/test-framework/ (props changed)
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/trunk/solr/build.xml (props changed)
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1380808&r1=1380807&r2=1380808&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue Sep 4 18:52:19 2012
@@ -138,6 +138,10 @@ Bug Fixes
* LUCENE-4333: Fixed NPE in TermGroupFacetCollector when faceting on mv fields.
(Jesse MacVicar, Martijn van Groningen)
+* NRTCachingDirectory was always caching a newly flushed segment in
+ RAM, instead of checking the estimated size of the segment
+ to decide whether to cache it. (Mike McCandless)
+
Optimizations
* LUCENE-4322: Decrease lucene-core JAR size. The core JAR size had increased a
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java?rev=1380808&r1=1380807&r2=1380808&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java Tue Sep 4 18:52:19 2012
@@ -267,9 +267,16 @@ public class NRTCachingDirectory extends
/** Subclass can override this to customize logic; return
* true if this file should be written to the RAMDirectory. */
protected boolean doCacheWrite(String name, IOContext context) {
- final MergeInfo merge = context.mergeInfo;
//System.out.println(Thread.currentThread().getName() + ": CACHE check merge=" + merge + " size=" + (merge==null ? 0 : merge.estimatedMergeBytes));
- return !name.equals(IndexFileNames.SEGMENTS_GEN) && (merge == null || merge.estimatedMergeBytes <= maxMergeSizeBytes) && cache.sizeInBytes() <= maxCachedBytes;
+
+ long bytes = 0;
+ if (context.mergeInfo != null) {
+ bytes = context.mergeInfo.estimatedMergeBytes;
+ } else if (context.flushInfo != null) {
+ bytes = context.flushInfo.estimatedSegmentSize;
+ }
+
+ return !name.equals(IndexFileNames.SEGMENTS_GEN) && (bytes <= maxMergeSizeBytes) && (bytes + cache.sizeInBytes()) <= maxCachedBytes;
}
private final Object uncacheLock = new Object();
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java?rev=1380808&r1=1380807&r2=1380808&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java Tue Sep 4 18:52:19 2012
@@ -38,12 +38,14 @@ import org.apache.lucene.codecs.TermStat
import org.apache.lucene.codecs.TermsConsumer;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util._TestUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -379,10 +381,13 @@ public class TestPostingsFormat extends
FieldInfos newFieldInfos = new FieldInfos(newFieldInfoArray);
+ // Estimate that flushed segment size will be 25% of
+ // what we use in RAM:
+ long bytes = RamUsageEstimator.sizeOf(fields)/4;
+
SegmentWriteState writeState = new SegmentWriteState(null, dir,
segmentInfo, newFieldInfos,
- 32, null, IOContext.DEFAULT);
-
+ 32, null, new IOContext(new FlushInfo(maxDocID, bytes)));
FieldsConsumer fieldsConsumer = Codec.getDefault().postingsFormat().fieldsConsumer(writeState);
for(Map.Entry<String,Map<BytesRef,List<Posting>>> fieldEnt : fields.entrySet()) {
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1380808&r1=1380807&r2=1380808&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Tue Sep 4 18:52:19 2012
@@ -443,7 +443,7 @@ public class MockDirectoryWrapper extend
}
//System.out.println(Thread.currentThread().getName() + ": MDW: create " + name);
- IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name, LuceneTestCase.newIOContext(randomState)), name);
+ IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name, LuceneTestCase.newIOContext(randomState, context)), name);
addFileHandle(io, name, Handle.Output);
openFilesForWrite.add(name);
@@ -497,7 +497,7 @@ public class MockDirectoryWrapper extend
throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false);
}
- IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name, LuceneTestCase.newIOContext(randomState)));
+ IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name, LuceneTestCase.newIOContext(randomState, context)));
addFileHandle(ii, name, Handle.Input);
return ii;
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1380808&r1=1380807&r2=1380808&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Tue Sep 4 18:52:19 2012
@@ -1130,10 +1130,25 @@ public abstract class LuceneTestCase ext
/** TODO: javadoc */
public static IOContext newIOContext(Random random) {
+ return newIOContext(random, IOContext.DEFAULT);
+ }
+
+ /** TODO: javadoc */
+ public static IOContext newIOContext(Random random, IOContext oldContext) {
final int randomNumDocs = random.nextInt(4192);
final int size = random.nextInt(512) * randomNumDocs;
- final IOContext context;
- switch (random.nextInt(5)) {
+ if (oldContext.flushInfo != null) {
+ // Always return at least the estimatedSegmentSize of
+ // the incoming IOContext:
+ return new IOContext(new FlushInfo(randomNumDocs, Math.max(oldContext.flushInfo.estimatedSegmentSize, size)));
+ } else if (oldContext.mergeInfo != null) {
+ // Always return at least the estimatedMergeBytes of
+ // the incoming IOContext:
+ return new IOContext(new MergeInfo(randomNumDocs, Math.max(oldContext.mergeInfo.estimatedMergeBytes, size), random.nextBoolean(), _TestUtil.nextInt(random, 1, 100)));
+ } else {
+ // Make a totally random IOContext:
+ final IOContext context;
+ switch (random.nextInt(5)) {
case 0:
context = IOContext.DEFAULT;
break;
@@ -1151,8 +1166,9 @@ public abstract class LuceneTestCase ext
break;
default:
context = IOContext.DEFAULT;
+ }
+ return context;
}
- return context;
}
/**