You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by mi...@apache.org on 2010/05/05 02:43:07 UTC
svn commit: r941124 - in /lucene/java/branches/lucene_3_0: ./ contrib/
contrib/highlighter/src/test/
contrib/instantiated/src/test/org/apache/lucene/store/instantiated/
src/java/org/apache/lucene/analysis/ src/java/org/apache/lucene/index/
src/java/org...
Author: mikemccand
Date: Wed May 5 00:43:06 2010
New Revision: 941124
URL: http://svn.apache.org/viewvc?rev=941124&view=rev
Log:
LUCENE-2283: merge to 30x
Modified:
lucene/java/branches/lucene_3_0/ (props changed)
lucene/java/branches/lucene_3_0/CHANGES.txt (contents, props changed)
lucene/java/branches/lucene_3_0/build.xml (props changed)
lucene/java/branches/lucene_3_0/contrib/ (props changed)
lucene/java/branches/lucene_3_0/contrib/CHANGES.txt (props changed)
lucene/java/branches/lucene_3_0/contrib/highlighter/src/test/ (props changed)
lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (props changed)
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/Tokenizer.java (props changed)
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/StoredFieldsWriter.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (props changed)
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/store/RAMFile.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/store/RAMOutputStream.java
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (props changed)
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java (props changed)
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java (props changed)
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/store/TestHugeRamFile.java
lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java (props changed)
Propchange: lucene/java/branches/lucene_3_0/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 5 00:43:06 2010
@@ -2,4 +2,4 @@
/lucene/java/branches/lucene_2_4:748824
/lucene/java/branches/lucene_2_9:817269-818600,825998,829134,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests:818601-821336
-/lucene/java/trunk:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399
+/lucene/java/trunk:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Modified: lucene/java/branches/lucene_3_0/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/CHANGES.txt?rev=941124&r1=941123&r2=941124&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/CHANGES.txt (original)
+++ lucene/java/branches/lucene_3_0/CHANGES.txt Wed May 5 00:43:06 2010
@@ -13,6 +13,12 @@ Bug fixes
in IndexWriter, nor the Reader in Tokenizer after close is
called. (Ruben Laguna, Uwe Schindler, Mike McCandless)
+* LUCENE-2283: Use shared memory pool for term vector and stored
+ fields buffers. This memory will be reclaimed if needed according to
+ the configured RAM Buffer Size for the IndexWriter. This also fixes
+ potentially excessive memory usage when many threads are indexing a
+ mix of small and large documents. (Tim Smith via Mike McCandless)
+
API Changes
* LUCENE-2281: added doBeforeFlush to IndexWriter to allow extensions to perform
Propchange: lucene/java/branches/lucene_3_0/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 5 00:43:06 2010
@@ -2,4 +2,4 @@
/lucene/java/branches/lucene_2_4/CHANGES.txt:748824
/lucene/java/branches/lucene_2_9/CHANGES.txt:817269-818600,825998,829134,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/CHANGES.txt:818601-821336
-/lucene/java/trunk/CHANGES.txt:881213,881315,881466,882374,882464,882672,882807,882888,882977,883074-883075,883554,883654,883661,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399
+/lucene/java/trunk/CHANGES.txt:881213,881315,881466,882374,882464,882672,882807,882888,882977,883074-883075,883554,883654,883661,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Propchange: lucene/java/branches/lucene_3_0/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 5 00:43:06 2010
@@ -2,4 +2,4 @@
/lucene/java/branches/lucene_2_4/build.xml:748824
/lucene/java/branches/lucene_2_9/build.xml:817269-818600,825998,829134,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/build.xml:818601-821336
-/lucene/java/trunk/build.xml:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887617,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399
+/lucene/java/trunk/build.xml:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887617,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Propchange: lucene/java/branches/lucene_3_0/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 5 00:43:06 2010
@@ -2,4 +2,4 @@
/lucene/java/branches/lucene_2_4/contrib:748824
/lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib:818601-821336
-/lucene/java/trunk/contrib:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399
+/lucene/java/trunk/contrib:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Propchange: lucene/java/branches/lucene_3_0/contrib/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 5 00:43:06 2010
@@ -2,4 +2,4 @@
/lucene/java/branches/lucene_2_4/contrib/CHANGES.txt:748824
/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib/CHANGES.txt:818601-821336
-/lucene/java/trunk/contrib/CHANGES.txt:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399
+/lucene/java/trunk/contrib/CHANGES.txt:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Propchange: lucene/java/branches/lucene_3_0/contrib/highlighter/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 5 00:43:06 2010
@@ -2,4 +2,4 @@
/lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824
/lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib/highlighter/src/test:818601-821336
-/lucene/java/trunk/contrib/highlighter/src/test:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399
+/lucene/java/trunk/contrib/highlighter/src/test:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Propchange: lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 5 00:43:06 2010
@@ -3,4 +3,4 @@
/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:817269-818600,825998,829134,829816,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:818601-821336
/lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:889463
-/lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399
+/lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Propchange: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/Tokenizer.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 5 00:43:06 2010
@@ -2,4 +2,4 @@
/lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/analysis/Tokenizer.java:748824
/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:817269-818600,825998,829134,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java:818601-821336
-/lucene/java/trunk/src/java/org/apache/lucene/analysis/Tokenizer.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,896661,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399
+/lucene/java/trunk/src/java/org/apache/lucene/analysis/Tokenizer.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,896661,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=941124&r1=941123&r2=941124&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java Wed May 5 00:43:06 2010
@@ -37,6 +37,7 @@ import org.apache.lucene.search.Similari
import org.apache.lucene.search.Weight;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMFile;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.ThreadInterruptedException;
@@ -171,6 +172,46 @@ final class DocumentsWriter {
this.next = next;
}
}
+
+ /**
+ * Create and return a new DocWriterBuffer.
+ */
+ PerDocBuffer newPerDocBuffer() {
+ return new PerDocBuffer();
+ }
+
+ /**
+ * RAMFile buffer for DocWriters.
+ */
+ class PerDocBuffer extends RAMFile {
+
+ /**
+ * Allocate bytes used from shared pool.
+ */
+ protected byte[] newBuffer(int size) {
+ assert size == PER_DOC_BLOCK_SIZE;
+ return perDocAllocator.getByteBlock(false);
+ }
+
+ /**
+ * Recycle the bytes used.
+ */
+ synchronized void recycle() {
+ if (buffers.size() > 0) {
+ setLength(0);
+
+ // Recycle the blocks
+ final int blockCount = buffers.size();
+
+ final byte[][] blocks = buffers.toArray( new byte[blockCount][] );
+ perDocAllocator.recycleByteBlocks(blocks, 0, blockCount);
+ buffers.clear();
+ sizeInBytes = 0;
+
+ assert numBuffers() == 0;
+ }
+ }
+ }
/**
* The IndexingChain must define the {@link #getChain(DocumentsWriter)} method
@@ -1186,6 +1227,11 @@ final class DocumentsWriter {
final static int BYTE_BLOCK_NOT_MASK = ~BYTE_BLOCK_MASK;
private class ByteBlockAllocator extends ByteBlockPool.Allocator {
+ final int blockSize;
+
+ ByteBlockAllocator(int blockSize) {
+ this.blockSize = blockSize;
+ }
ArrayList<byte[]> freeByteBlocks = new ArrayList<byte[]>();
@@ -1202,12 +1248,12 @@ final class DocumentsWriter {
// things that don't track allocations (term
// vectors) and things that do (freq/prox
// postings).
- numBytesAlloc += BYTE_BLOCK_SIZE;
- b = new byte[BYTE_BLOCK_SIZE];
+ numBytesAlloc += blockSize;
+ b = new byte[blockSize];
} else
b = freeByteBlocks.remove(size-1);
if (trackAllocations)
- numBytesUsed += BYTE_BLOCK_SIZE;
+ numBytesUsed += blockSize;
assert numBytesUsed <= numBytesAlloc;
return b;
}
@@ -1268,7 +1314,12 @@ final class DocumentsWriter {
freeIntBlocks.add(blocks[i]);
}
- ByteBlockAllocator byteBlockAllocator = new ByteBlockAllocator();
+ ByteBlockAllocator byteBlockAllocator = new ByteBlockAllocator(BYTE_BLOCK_SIZE);
+
+ final static int PER_DOC_BLOCK_SIZE = 1024;
+
+ final ByteBlockAllocator perDocAllocator = new ByteBlockAllocator(PER_DOC_BLOCK_SIZE);
+
/* Initial chunk size of the shared char[] blocks used to
store term text */
@@ -1308,10 +1359,12 @@ final class DocumentsWriter {
return nf.format(v/1024./1024.);
}
- /* We have three pools of RAM: Postings, byte blocks
- * (holds freq/prox posting data) and char blocks (holds
- * characters in the term). Different docs require
- * varying amount of storage from these three classes.
+ /* We have four pools of RAM: Postings, byte blocks
+ * (holds freq/prox posting data), char blocks (holds
+ * characters in the term) and per-doc buffers (stored fields/term vectors).
+ * Different docs require varying amount of storage from
+ * these four classes.
+ *
* For example, docs with many unique single-occurrence
* short terms will use up the Postings RAM and hardly any
* of the other two. Whereas docs with very large terms
@@ -1335,6 +1388,7 @@ final class DocumentsWriter {
" deletesMB=" + toMB(deletesRAMUsed) +
" vs trigger=" + toMB(freeTrigger) +
" byteBlockFree=" + toMB(byteBlockAllocator.freeByteBlocks.size()*BYTE_BLOCK_SIZE) +
+ " perDocFree=" + toMB(perDocAllocator.freeByteBlocks.size()*PER_DOC_BLOCK_SIZE) +
" charBlockFree=" + toMB(freeCharBlocks.size()*CHAR_BLOCK_SIZE*CHAR_NUM_BYTE));
final long startBytesAlloc = numBytesAlloc + deletesRAMUsed;
@@ -1350,7 +1404,11 @@ final class DocumentsWriter {
while(numBytesAlloc+deletesRAMUsed > freeLevel) {
synchronized(this) {
- if (0 == byteBlockAllocator.freeByteBlocks.size() && 0 == freeCharBlocks.size() && 0 == freeIntBlocks.size() && !any) {
+ if (0 == perDocAllocator.freeByteBlocks.size()
+ && 0 == byteBlockAllocator.freeByteBlocks.size()
+ && 0 == freeCharBlocks.size()
+ && 0 == freeIntBlocks.size()
+ && !any) {
// Nothing else to free -- must flush now.
bufferIsFull = numBytesUsed+deletesRAMUsed > flushTrigger;
if (infoStream != null) {
@@ -1363,23 +1421,34 @@ final class DocumentsWriter {
break;
}
- if ((0 == iter % 4) && byteBlockAllocator.freeByteBlocks.size() > 0) {
+ if ((0 == iter % 5) && byteBlockAllocator.freeByteBlocks.size() > 0) {
byteBlockAllocator.freeByteBlocks.remove(byteBlockAllocator.freeByteBlocks.size()-1);
numBytesAlloc -= BYTE_BLOCK_SIZE;
}
- if ((1 == iter % 4) && freeCharBlocks.size() > 0) {
+ if ((1 == iter % 5) && freeCharBlocks.size() > 0) {
freeCharBlocks.remove(freeCharBlocks.size()-1);
numBytesAlloc -= CHAR_BLOCK_SIZE * CHAR_NUM_BYTE;
}
- if ((2 == iter % 4) && freeIntBlocks.size() > 0) {
+ if ((2 == iter % 5) && freeIntBlocks.size() > 0) {
freeIntBlocks.remove(freeIntBlocks.size()-1);
numBytesAlloc -= INT_BLOCK_SIZE * INT_NUM_BYTE;
}
+
+ if ((3 == iter % 5) && perDocAllocator.freeByteBlocks.size() > 0) {
+ // Remove upwards of 32 blocks (each block is 1K)
+ for (int i = 0; i < 32; ++i) {
+ perDocAllocator.freeByteBlocks.remove(perDocAllocator.freeByteBlocks.size() - 1);
+ numBytesAlloc -= PER_DOC_BLOCK_SIZE;
+ if (perDocAllocator.freeByteBlocks.size() == 0) {
+ break;
+ }
+ }
+ }
}
- if ((3 == iter % 4) && any)
+ if ((4 == iter % 5) && any)
// Ask consumer to free any recycled state
any = consumer.freeRAM();
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/StoredFieldsWriter.java?rev=941124&r1=941123&r2=941124&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/StoredFieldsWriter.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/StoredFieldsWriter.java Wed May 5 00:43:06 2010
@@ -166,14 +166,13 @@ final class StoredFieldsWriter {
}
class PerDoc extends DocumentsWriter.DocWriter {
-
- // TODO: use something more memory efficient; for small
- // docs the 1024 buffer size of RAMOutputStream wastes alot
- RAMOutputStream fdt = new RAMOutputStream();
+ final DocumentsWriter.PerDocBuffer buffer = docWriter.newPerDocBuffer();
+ RAMOutputStream fdt = new RAMOutputStream(buffer);
int numStoredFields;
void reset() {
fdt.reset();
+ buffer.recycle();
numStoredFields = 0;
}
@@ -185,7 +184,7 @@ final class StoredFieldsWriter {
@Override
public long sizeInBytes() {
- return fdt.sizeInBytes();
+ return buffer.getSizeInBytes();
}
@Override
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=941124&r1=941123&r2=941124&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java Wed May 5 00:43:06 2010
@@ -192,8 +192,8 @@ final class TermVectorsTermsWriter exten
tvd.writeVLong(pos-lastPos);
lastPos = pos;
}
- perDoc.tvf.writeTo(tvf);
- perDoc.tvf.reset();
+ perDoc.perDocTvf.writeTo(tvf);
+ perDoc.perDocTvf.reset();
perDoc.numVectorFields = 0;
}
@@ -244,16 +244,17 @@ final class TermVectorsTermsWriter exten
class PerDoc extends DocumentsWriter.DocWriter {
- // TODO: use something more memory efficient; for small
- // docs the 1024 buffer size of RAMOutputStream wastes alot
- RAMOutputStream tvf = new RAMOutputStream();
+ final DocumentsWriter.PerDocBuffer buffer = docWriter.newPerDocBuffer();
+ RAMOutputStream perDocTvf = new RAMOutputStream(buffer);
+
int numVectorFields;
int[] fieldNumbers = new int[1];
long[] fieldPointers = new long[1];
void reset() {
- tvf.reset();
+ perDocTvf.reset();
+ buffer.recycle();
numVectorFields = 0;
}
@@ -269,13 +270,13 @@ final class TermVectorsTermsWriter exten
fieldPointers = ArrayUtil.grow(fieldPointers);
}
fieldNumbers[numVectorFields] = fieldNumber;
- fieldPointers[numVectorFields] = tvf.getFilePointer();
+ fieldPointers[numVectorFields] = perDocTvf.getFilePointer();
numVectorFields++;
}
@Override
public long sizeInBytes() {
- return tvf.sizeInBytes();
+ return buffer.getSizeInBytes();
}
@Override
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java?rev=941124&r1=941123&r2=941124&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java Wed May 5 00:43:06 2010
@@ -74,8 +74,8 @@ final class TermVectorsTermsWriterPerFie
perThread.doc = termsWriter.getPerDoc();
perThread.doc.docID = docState.docID;
assert perThread.doc.numVectorFields == 0;
- assert 0 == perThread.doc.tvf.length();
- assert 0 == perThread.doc.tvf.getFilePointer();
+ assert 0 == perThread.doc.perDocTvf.length();
+ assert 0 == perThread.doc.perDocTvf.getFilePointer();
} else {
assert perThread.doc.docID == docState.docID;
@@ -114,7 +114,7 @@ final class TermVectorsTermsWriterPerFie
if (numPostings > maxNumPostings)
maxNumPostings = numPostings;
- final IndexOutput tvf = perThread.doc.tvf;
+ final IndexOutput tvf = perThread.doc.perDocTvf;
// This is called once, after inverting all occurrences
// of a given field in the doc. At this point we flush
Propchange: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 5 00:43:06 2010
@@ -2,4 +2,4 @@
/lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:748824
/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:817269-818600,825998,829134,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:818601-821336
-/lucene/java/trunk/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:881213,881315,881466,881984,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399
+/lucene/java/trunk/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:881213,881315,881466,881984,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/store/RAMFile.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/store/RAMFile.java?rev=941124&r1=941123&r2=941124&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/store/RAMFile.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/store/RAMFile.java Wed May 5 00:43:06 2010
@@ -20,44 +20,45 @@ package org.apache.lucene.store;
import java.util.ArrayList;
import java.io.Serializable;
-class RAMFile implements Serializable {
+/** For Lucene internal use */
+public class RAMFile implements Serializable {
private static final long serialVersionUID = 1l;
- private ArrayList<byte[]> buffers = new ArrayList<byte[]>();
+ protected ArrayList<byte[]> buffers = new ArrayList<byte[]>();
long length;
RAMDirectory directory;
- long sizeInBytes;
+ protected long sizeInBytes;
// This is publicly modifiable via Directory.touchFile(), so direct access not supported
private long lastModified = System.currentTimeMillis();
// File used as buffer, in no RAMDirectory
- RAMFile() {}
+ protected RAMFile() {}
RAMFile(RAMDirectory directory) {
this.directory = directory;
}
// For non-stream access from thread that might be concurrent with writing
- synchronized long getLength() {
+ public synchronized long getLength() {
return length;
}
- synchronized void setLength(long length) {
+ protected synchronized void setLength(long length) {
this.length = length;
}
// For non-stream access from thread that might be concurrent with writing
- synchronized long getLastModified() {
+ public synchronized long getLastModified() {
return lastModified;
}
- synchronized void setLastModified(long lastModified) {
+ protected synchronized void setLastModified(long lastModified) {
this.lastModified = lastModified;
}
- final byte[] addBuffer(int size) {
+ protected final byte[] addBuffer(int size) {
byte[] buffer = newBuffer(size);
synchronized(this) {
buffers.add(buffer);
@@ -72,11 +73,11 @@ class RAMFile implements Serializable {
return buffer;
}
- final synchronized byte[] getBuffer(int index) {
+ protected final synchronized byte[] getBuffer(int index) {
return buffers.get(index);
}
- final synchronized int numBuffers() {
+ protected final synchronized int numBuffers() {
return buffers.size();
}
@@ -86,14 +87,12 @@ class RAMFile implements Serializable {
* @param size size of allocated buffer.
* @return allocated buffer.
*/
- byte[] newBuffer(int size) {
+ protected byte[] newBuffer(int size) {
return new byte[size];
}
- synchronized long getSizeInBytes() {
- synchronized(directory) {
- return sizeInBytes;
- }
+ public synchronized long getSizeInBytes() {
+ return sizeInBytes;
}
}
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/store/RAMOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/store/RAMOutputStream.java?rev=941124&r1=941123&r2=941124&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/store/RAMOutputStream.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/store/RAMOutputStream.java Wed May 5 00:43:06 2010
@@ -21,6 +21,8 @@ import java.io.IOException;
/**
* A memory-resident {@link IndexOutput} implementation.
+ *
+ * <p>For Lucene internal use.</p>
*/
public class RAMOutputStream extends IndexOutput {
@@ -40,7 +42,7 @@ public class RAMOutputStream extends Ind
this(new RAMFile());
}
- RAMOutputStream(RAMFile f) {
+ public RAMOutputStream(RAMFile f) {
file = f;
// make sure that we switch to the
@@ -66,14 +68,13 @@ public class RAMOutputStream extends Ind
}
}
- /** Resets this to an empty buffer. */
+ /** Resets this to an empty file. */
public void reset() {
- try {
- seek(0);
- } catch (IOException e) { // should never happen
- throw new RuntimeException(e.toString());
- }
-
+ currentBuffer = null;
+ currentBufferIndex = -1;
+ bufferPosition = 0;
+ bufferStart = 0;
+ bufferLength = 0;
file.setLength(0);
}
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 5 00:43:06 2010
@@ -1,4 +1,4 @@
/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:932398
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:825998,829134,829881,831036,896850,909334,940987
-/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399
+/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 5 00:43:06 2010
@@ -1,4 +1,4 @@
/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:932398
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:825998,829134,829881,831036,896850,909334,940987
-/lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399
+/lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 5 00:43:06 2010
@@ -1,4 +1,4 @@
/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:932398
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:825998,829134,829881,831036,896850,909334,940987
-/lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399
+/lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 5 00:43:06 2010
@@ -1,4 +1,4 @@
/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:932398
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:825998,829134,829881,831036,896850,909334,940987
-/lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399
+/lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060
Modified: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/store/TestHugeRamFile.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/store/TestHugeRamFile.java?rev=941124&r1=941123&r2=941124&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/store/TestHugeRamFile.java (original)
+++ lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/store/TestHugeRamFile.java Wed May 5 00:43:06 2010
@@ -33,7 +33,7 @@ public class TestHugeRamFile extends Luc
private long capacity = 0;
private HashMap singleBuffers = new HashMap();
@Override
- byte[] newBuffer(int size) {
+ protected byte[] newBuffer(int size) {
capacity += size;
if (capacity <= MAX_VALUE) {
// below maxint we reuse buffers
Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 5 00:43:06 2010
@@ -2,4 +2,4 @@
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/util/TestAttributeSource.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:817269-818600,825998,829134,829881,831036,896850,909334,940987
/lucene/java/branches/lucene_2_9_back_compat_tests/src/test/org/apache/lucene/util/TestAttributeSource.java:818601-821336
-/lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883079,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399
+/lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883079,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060