You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by mi...@apache.org on 2010/05/18 14:12:03 UTC
svn commit: r945608 - in /lucene/java/branches/lucene_2_9: ./
src/java/org/apache/lucene/index/
Author: mikemccand
Date: Tue May 18 12:12:03 2010
New Revision: 945608
URL: http://svn.apache.org/viewvc?rev=945608&view=rev
Log:
LUCENE-2467: fix memory leaks in IW when indexing very large docs
Modified:
lucene/java/branches/lucene_2_9/CHANGES.txt
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/ReusableStringReader.java
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHash.java
Modified: lucene/java/branches/lucene_2_9/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/CHANGES.txt?rev=945608&r1=945607&r2=945608&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/CHANGES.txt (original)
+++ lucene/java/branches/lucene_2_9/CHANGES.txt Tue May 18 12:12:03 2010
@@ -20,6 +20,9 @@ Bug fixes
indexing a mix of small and large documents. (Tim Smith via Mike
McCandless)
+ * LUCENE-2467: Fixed memory leaks in IndexWriter when large documents
+ are indexed. (Mike McCandless)
+
======================= Release 2.9.2 2010-02-26 =======================
Bug fixes
Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=945608&r1=945607&r2=945608&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java Tue May 18 12:12:03 2010
@@ -1276,8 +1276,13 @@ final class DocumentsWriter {
/* Return byte[]'s to the pool */
void recycleByteBlocks(byte[][] blocks, int start, int end) {
synchronized(DocumentsWriter.this) {
- for(int i=start;i<end;i++)
+ for(int i=start;i<end;i++) {
freeByteBlocks.add(blocks[i]);
+ blocks[i] = null;
+ }
+ if (infoStream != null && blockSize != 1024) {
+ message("DW.recycleByteBlocks blockSize=" + blockSize + " count=" + (end-start) + " total now " + freeByteBlocks.size());
+ }
}
}
}
@@ -1313,7 +1318,6 @@ final class DocumentsWriter {
synchronized void bytesAllocated(long numBytes) {
numBytesAlloc += numBytes;
- assert numBytesUsed <= numBytesAlloc;
}
synchronized void bytesUsed(long numBytes) {
@@ -1323,8 +1327,13 @@ final class DocumentsWriter {
/* Return int[]s to the pool */
synchronized void recycleIntBlocks(int[][] blocks, int start, int end) {
- for(int i=start;i<end;i++)
+ for(int i=start;i<end;i++) {
freeIntBlocks.add(blocks[i]);
+ blocks[i] = null;
+ }
+ if (infoStream != null) {
+ message("DW.recycleIntBlocks count=" + (end-start) + " total now " + freeIntBlocks.size());
+ }
}
ByteBlockAllocator byteBlockAllocator = new ByteBlockAllocator(BYTE_BLOCK_SIZE);
@@ -1364,8 +1373,13 @@ final class DocumentsWriter {
/* Return char[]s to the pool */
synchronized void recycleCharBlocks(char[][] blocks, int numBlocks) {
- for(int i=0;i<numBlocks;i++)
+ for(int i=0;i<numBlocks;i++) {
freeCharBlocks.add(blocks[i]);
+ blocks[i] = null;
+ }
+ if (infoStream != null) {
+ message("DW.recycleCharBlocks count=" + numBlocks + " total now " + freeCharBlocks.size());
+ }
}
String toMB(long v) {
@@ -1425,7 +1439,7 @@ final class DocumentsWriter {
// Nothing else to free -- must flush now.
bufferIsFull = numBytesUsed+deletesRAMUsed > flushTrigger;
if (infoStream != null) {
- if (numBytesUsed > flushTrigger)
+ if (bufferIsFull)
message(" nothing to free; now set bufferIsFull");
else
message(" nothing to free");
Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java?rev=945608&r1=945607&r2=945608&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java Tue May 18 12:12:03 2010
@@ -4208,6 +4208,9 @@ public class IndexWriter {
try {
return doFlushInternal(flushDocStores, flushDeletes);
} finally {
+ if (docWriter.doBalanceRAM()) {
+ docWriter.balanceRAM();
+ }
docWriter.clearFlushPending();
}
}
Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/ReusableStringReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/ReusableStringReader.java?rev=945608&r1=945607&r2=945608&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/ReusableStringReader.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/ReusableStringReader.java Tue May 18 12:12:03 2010
@@ -41,6 +41,7 @@ final class ReusableStringReader extends
left -= len;
return len;
} else if (0 == left) {
+ s = null;
return -1;
} else {
s.getChars(upto, upto+left, c, off);
Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=945608&r1=945607&r2=945608&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java Tue May 18 12:12:03 2010
@@ -192,7 +192,6 @@ final class TermVectorsTermsWriter exten
lastPos = pos;
}
perDoc.perDocTvf.writeTo(tvf);
- perDoc.perDocTvf.reset();
perDoc.numVectorFields = 0;
}
@@ -200,6 +199,7 @@ final class TermVectorsTermsWriter exten
lastDocID++;
+ perDoc.reset();
free(perDoc);
assert docWriter.writer.testPoint("TermVectorsTermsWriter.finishDocument end");
}
Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHash.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHash.java?rev=945608&r1=945607&r2=945608&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHash.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHash.java Tue May 18 12:12:03 2010
@@ -89,8 +89,17 @@ final class TermsHash extends InvertedDo
assert postingsFreeCount == postingsAllocCount: Thread.currentThread().getName() + ": postingsFreeCount=" + postingsFreeCount + " postingsAllocCount=" + postingsAllocCount + " consumer=" + consumer;
- final int newSize = ArrayUtil.getShrinkSize(postingsFreeList.length, postingsAllocCount);
+ final int newSize = 1;
+
if (newSize != postingsFreeList.length) {
+ if (postingsFreeCount > newSize) {
+ if (trackAllocations) {
+ docWriter.bytesAllocated(-(postingsFreeCount-newSize) * bytesPerPosting);
+ }
+ postingsFreeCount = newSize;
+ postingsAllocCount = newSize;
+ }
+
RawPostingList[] newArray = new RawPostingList[newSize];
System.arraycopy(postingsFreeList, 0, newArray, 0, postingsFreeCount);
postingsFreeList = newArray;