You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2010/05/18 01:12:34 UTC
svn commit: r945420 - in /lucene/dev/trunk/lucene: ./
src/java/org/apache/lucene/index/
Author: mikemccand
Date: Mon May 17 23:12:34 2010
New Revision: 945420
URL: http://svn.apache.org/viewvc?rev=945420&view=rev
Log:
LUCENE-2467: fix memory leaks in IW when indexing very large docs
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ReusableStringReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=945420&r1=945419&r2=945420&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Mon May 17 23:12:34 2010
@@ -391,6 +391,9 @@ Bug fixes
addition, all of IndexComnmit methods which threw
UnsupportedOperationException are now abstract. (Shai Erera)
+* LUCENE-2467: Fixed memory leaks in IndexWriter when large documents
+ are indexed. (Mike McCandless)
+
New features
* LUCENE-2128: Parallelized fetching document frequencies during weight
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=945420&r1=945419&r2=945420&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Mon May 17 23:12:34 2010
@@ -1314,11 +1314,14 @@ final class DocumentsWriter {
}
/* Return byte[]'s to the pool */
+
@Override
void recycleByteBlocks(byte[][] blocks, int start, int end) {
synchronized(DocumentsWriter.this) {
- for(int i=start;i<end;i++)
+ for(int i=start;i<end;i++) {
freeByteBlocks.add(blocks[i]);
+ blocks[i] = null;
+ }
}
}
@@ -1326,9 +1329,11 @@ final class DocumentsWriter {
void recycleByteBlocks(List<byte[]> blocks) {
synchronized(DocumentsWriter.this) {
final int size = blocks.size();
- for(int i=0;i<size;i++)
+ for(int i=0;i<size;i++) {
freeByteBlocks.add(blocks.get(i));
- }
+ blocks.set(i, null);
+ }
+ }
}
}
@@ -1358,8 +1363,10 @@ final class DocumentsWriter {
/* Return int[]s to the pool */
synchronized void recycleIntBlocks(int[][] blocks, int start, int end) {
- for(int i=start;i<end;i++)
+ for(int i=start;i<end;i++) {
freeIntBlocks.add(blocks[i]);
+ blocks[i] = null;
+ }
}
ByteBlockAllocator byteBlockAllocator = new ByteBlockAllocator(BYTE_BLOCK_SIZE);
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=945420&r1=945419&r2=945420&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java Mon May 17 23:12:34 2010
@@ -3624,7 +3624,11 @@ public class IndexWriter implements Clos
// even while a flush is happening
private synchronized final boolean doFlush(boolean flushDocStores, boolean flushDeletes) throws CorruptIndexException, IOException {
try {
- return doFlushInternal(flushDocStores, flushDeletes);
+ try {
+ return doFlushInternal(flushDocStores, flushDeletes);
+ } finally {
+ docWriter.balanceRAM();
+ }
} finally {
docWriter.clearFlushPending();
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ReusableStringReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ReusableStringReader.java?rev=945420&r1=945419&r2=945420&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ReusableStringReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ReusableStringReader.java Mon May 17 23:12:34 2010
@@ -43,6 +43,8 @@ final class ReusableStringReader extends
left -= len;
return len;
} else if (0 == left) {
+ // don't keep a reference (s could have been very large)
+ s = null;
return -1;
} else {
s.getChars(upto, upto+left, c, off);
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=945420&r1=945419&r2=945420&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java Mon May 17 23:12:34 2010
@@ -190,7 +190,6 @@ final class TermVectorsTermsWriter exten
lastPos = pos;
}
perDoc.perDocTvf.writeTo(tvf);
- perDoc.perDocTvf.reset();
perDoc.numVectorFields = 0;
}
@@ -198,6 +197,7 @@ final class TermVectorsTermsWriter exten
lastDocID++;
+ perDoc.reset();
free(perDoc);
assert docWriter.writer.testPoint("TermVectorsTermsWriter.finishDocument end");
}