You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by mi...@apache.org on 2010/05/18 11:32:07 UTC
svn commit: r945556 - in /lucene/java/branches/lucene_3_0: ./ contrib/
contrib/highlighter/src/test/
contrib/instantiated/src/test/org/apache/lucene/store/instantiated/
src/java/org/apache/lucene/analysis/ src/java/org/apache/lucene/index/
src/java/org...
Author: mikemccand
Date: Tue May 18 09:32:07 2010
New Revision: 945556
URL: http://svn.apache.org/viewvc?rev=945556&view=rev
Log:
LUCENE-2467: fix memory leaks in IW when indexing very large docs
Modified:
lucene/java/branches/lucene_3_0/ (props changed)
lucene/java/branches/lucene_3_0/CHANGES.txt (contents, props changed)
lucene/java/branches/lucene_3_0/build.xml (props changed)
lucene/java/branches/lucene_3_0/contrib/ (props changed)
lucene/java/branches/lucene_3_0/contrib/CHANGES.txt (props changed)
lucene/java/branches/lucene_3_0/contrib/highlighter/src/test/ (props changed)
lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (props changed)
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/Tokenizer.java (props changed)
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/ReusableStringReader.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermsHash.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (props changed)
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (props changed)
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java (props changed)
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java (props changed)
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java (props changed)
Propchange: lucene/java/branches/lucene_3_0/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene:932398
+/lucene/dev/trunk/lucene:932398,945420
/lucene/java/branches/lucene_2_4:748824
/lucene/java/branches/lucene_2_9:817269-818600,825998,829134,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests:818601-821336
Modified: lucene/java/branches/lucene_3_0/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/CHANGES.txt?rev=945556&r1=945555&r2=945556&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/CHANGES.txt (original)
+++ lucene/java/branches/lucene_3_0/CHANGES.txt Tue May 18 09:32:07 2010
@@ -19,6 +19,9 @@ Bug fixes
potentially excessive memory usage when many threads are indexing a
mix of small and large documents. (Tim Smith via Mike McCandless)
+* LUCENE-2467: Fixed memory leaks in IndexWriter when large documents
+ are indexed. (Mike McCandless)
+
API Changes
* LUCENE-2281: added doBeforeFlush to IndexWriter to allow extensions to perform
Propchange: lucene/java/branches/lucene_3_0/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/CHANGES.txt:932398
+/lucene/dev/trunk/lucene/CHANGES.txt:932398,945420
/lucene/java/branches/lucene_2_4/CHANGES.txt:748824
/lucene/java/branches/lucene_2_9/CHANGES.txt:817269-818600,825998,829134,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/CHANGES.txt:818601-821336
Propchange: lucene/java/branches/lucene_3_0/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/build.xml:932398
+/lucene/dev/trunk/lucene/build.xml:932398,945420
/lucene/java/branches/lucene_2_4/build.xml:748824
/lucene/java/branches/lucene_2_9/build.xml:817269-818600,825998,829134,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/build.xml:818601-821336
Propchange: lucene/java/branches/lucene_3_0/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/contrib:932398
+/lucene/dev/trunk/lucene/contrib:932398,945420
/lucene/java/branches/lucene_2_4/contrib:748824
/lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib:818601-821336
Propchange: lucene/java/branches/lucene_3_0/contrib/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/contrib/CHANGES.txt:932398
+/lucene/dev/trunk/lucene/contrib/CHANGES.txt:932398,945420
/lucene/java/branches/lucene_2_4/contrib/CHANGES.txt:748824
/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib/CHANGES.txt:818601-821336
Propchange: lucene/java/branches/lucene_3_0/contrib/highlighter/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/contrib/highlighter/src/test:932398
+/lucene/dev/trunk/lucene/contrib/highlighter/src/test:932398,945420
/lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824
/lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib/highlighter/src/test:818601-821336
Propchange: lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:932398
+/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:932398,945420
/lucene/java/branches/lucene_2_4/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:748824
/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:817269-818600,825998,829134,829816,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:818601-821336
Propchange: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/Tokenizer.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:932398
+/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:932398,945420
/lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/analysis/Tokenizer.java:748824
/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:817269-818600,825998,829134,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java:818601-821336
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=945556&r1=945555&r2=945556&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java Tue May 18 09:32:07 2010
@@ -1260,11 +1260,14 @@ final class DocumentsWriter {
}
/* Return byte[]'s to the pool */
+
@Override
void recycleByteBlocks(byte[][] blocks, int start, int end) {
synchronized(DocumentsWriter.this) {
- for(int i=start;i<end;i++)
+ for(int i=start;i<end;i++) {
freeByteBlocks.add(blocks[i]);
+ blocks[i] = null;
+ }
}
}
}
@@ -1300,7 +1303,6 @@ final class DocumentsWriter {
synchronized void bytesAllocated(long numBytes) {
numBytesAlloc += numBytes;
- assert numBytesUsed <= numBytesAlloc;
}
synchronized void bytesUsed(long numBytes) {
@@ -1310,8 +1312,10 @@ final class DocumentsWriter {
/* Return int[]s to the pool */
synchronized void recycleIntBlocks(int[][] blocks, int start, int end) {
- for(int i=start;i<end;i++)
+ for(int i=start;i<end;i++) {
freeIntBlocks.add(blocks[i]);
+ blocks[i] = null;
+ }
}
ByteBlockAllocator byteBlockAllocator = new ByteBlockAllocator(BYTE_BLOCK_SIZE);
@@ -1351,8 +1355,10 @@ final class DocumentsWriter {
/* Return char[]s to the pool */
synchronized void recycleCharBlocks(char[][] blocks, int numBlocks) {
- for(int i=0;i<numBlocks;i++)
+ for(int i=0;i<numBlocks;i++) {
freeCharBlocks.add(blocks[i]);
+ blocks[i] = null;
+ }
}
String toMB(long v) {
@@ -1412,7 +1418,7 @@ final class DocumentsWriter {
// Nothing else to free -- must flush now.
bufferIsFull = numBytesUsed+deletesRAMUsed > flushTrigger;
if (infoStream != null) {
- if (numBytesUsed > flushTrigger)
+ if (bufferIsFull)
message(" nothing to free; now set bufferIsFull");
else
message(" nothing to free");
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java?rev=945556&r1=945555&r2=945556&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java Tue May 18 09:32:07 2010
@@ -3512,7 +3512,13 @@ public class IndexWriter implements Clos
// even while a flush is happening
private synchronized final boolean doFlush(boolean flushDocStores, boolean flushDeletes) throws CorruptIndexException, IOException {
try {
- return doFlushInternal(flushDocStores, flushDeletes);
+ try {
+ return doFlushInternal(flushDocStores, flushDeletes);
+ } finally {
+ if (docWriter.doBalanceRAM()) {
+ docWriter.balanceRAM();
+ }
+ }
} finally {
docWriter.clearFlushPending();
}
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/ReusableStringReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/ReusableStringReader.java?rev=945556&r1=945555&r2=945556&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/ReusableStringReader.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/ReusableStringReader.java Tue May 18 09:32:07 2010
@@ -43,6 +43,8 @@ final class ReusableStringReader extends
left -= len;
return len;
} else if (0 == left) {
+ // don't keep a reference (s could have been very large)
+ s = null;
return -1;
} else {
s.getChars(upto, upto+left, c, off);
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=945556&r1=945555&r2=945556&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java Tue May 18 09:32:07 2010
@@ -193,7 +193,6 @@ final class TermVectorsTermsWriter exten
lastPos = pos;
}
perDoc.perDocTvf.writeTo(tvf);
- perDoc.perDocTvf.reset();
perDoc.numVectorFields = 0;
}
@@ -201,6 +200,7 @@ final class TermVectorsTermsWriter exten
lastDocID++;
+ perDoc.reset();
free(perDoc);
assert docWriter.writer.testPoint("TermVectorsTermsWriter.finishDocument end");
}
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermsHash.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermsHash.java?rev=945556&r1=945555&r2=945556&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermsHash.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermsHash.java Tue May 18 09:32:07 2010
@@ -89,8 +89,17 @@ final class TermsHash extends InvertedDo
assert postingsFreeCount == postingsAllocCount: Thread.currentThread().getName() + ": postingsFreeCount=" + postingsFreeCount + " postingsAllocCount=" + postingsAllocCount + " consumer=" + consumer;
- final int newSize = ArrayUtil.getShrinkSize(postingsFreeList.length, postingsAllocCount);
+ final int newSize = 1;
if (newSize != postingsFreeList.length) {
+ if (postingsFreeCount > newSize) {
+ if (trackAllocations) {
+ docWriter.bytesAllocated(-(postingsFreeCount-newSize) * bytesPerPosting);
+ }
+
+ postingsFreeCount = newSize;
+ postingsAllocCount = newSize;
+ }
+
RawPostingList[] newArray = new RawPostingList[newSize];
System.arraycopy(postingsFreeList, 0, newArray, 0, postingsFreeCount);
postingsFreeList = newArray;
Propchange: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:932398
+/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:932398,945420
/lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:748824
/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:817269-818600,825998,829134,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:818601-821336
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:932398
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:932398,945420
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:825998,829134,829881,831036,896850,909334,940987
/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:932398
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:932398,945420
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:825998,829134,829881,831036,896850,909334,940987
/lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:932398
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:932398,945420
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:825998,829134,829881,831036,896850,909334,940987
/lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:932398
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:932398,945420
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:825998,829134,829881,831036,896850,909334,940987
/lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:932398
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:932398,945420
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/util/TestAttributeSource.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:817269-818600,825998,829134,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/src/test/org/apache/lucene/util/TestAttributeSource.java:818601-821336