You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by mi...@apache.org on 2008/05/17 18:49:18 UTC
svn commit: r657397 - in /lucene/java/trunk/src:
java/org/apache/lucene/index/ test/org/apache/lucene/index/
Author: mikemccand
Date: Sat May 17 09:49:18 2008
New Revision: 657397
URL: http://svn.apache.org/viewvc?rev=657397&view=rev
Log:
LUCENE-1283: factor out ByteSliceWriter from DocumentsWriter
Added:
lucene/java/trunk/src/java/org/apache/lucene/index/ByteSliceWriter.java (with props)
lucene/java/trunk/src/test/org/apache/lucene/index/TestByteSlices.java (with props)
Modified:
lucene/java/trunk/src/java/org/apache/lucene/index/ByteBlockPool.java
lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriterFieldData.java
lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriterThreadState.java
lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing2.java
Modified: lucene/java/trunk/src/java/org/apache/lucene/index/ByteBlockPool.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/ByteBlockPool.java?rev=657397&r1=657396&r2=657397&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/ByteBlockPool.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/ByteBlockPool.java Sat May 17 09:49:18 2008
@@ -37,6 +37,11 @@
final class ByteBlockPool {
+ abstract static class Allocator {
+ abstract void recycleByteBlocks(byte[][] blocks, int start, int end);
+ abstract byte[] getByteBlock(boolean trackAllocations);
+ }
+
public byte[][] buffers = new byte[10][];
int bufferUpto = -1; // Which buffer we are upto
@@ -45,11 +50,11 @@
public byte[] buffer; // Current head buffer
public int byteOffset = -DocumentsWriter.BYTE_BLOCK_SIZE; // Current head offset
- private boolean trackAllocations;
- DocumentsWriter docWriter;
+ private final boolean trackAllocations;
+ private final Allocator allocator;
- public ByteBlockPool(DocumentsWriter docWriter, boolean trackAllocations) {
- this.docWriter = docWriter;
+ public ByteBlockPool(Allocator allocator, boolean trackAllocations) {
+ this.allocator = allocator;
this.trackAllocations = trackAllocations;
}
@@ -66,7 +71,7 @@
if (bufferUpto > 0)
// Recycle all but the first buffer
- docWriter.recycleByteBlocks(buffers, 1, 1+bufferUpto);
+ allocator.recycleByteBlocks(buffers, 1, 1+bufferUpto);
// Re-use the first buffer
bufferUpto = 0;
@@ -82,7 +87,7 @@
System.arraycopy(buffers, 0, newBuffers, 0, buffers.length);
buffers = newBuffers;
}
- buffer = buffers[1+bufferUpto] = docWriter.getByteBlock(trackAllocations);
+ buffer = buffers[1+bufferUpto] = allocator.getByteBlock(trackAllocations);
bufferUpto++;
byteUpto = 0;
Added: lucene/java/trunk/src/java/org/apache/lucene/index/ByteSliceWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/ByteSliceWriter.java?rev=657397&view=auto
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/ByteSliceWriter.java (added)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/ByteSliceWriter.java Sat May 17 09:49:18 2008
@@ -0,0 +1,89 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * Class to write byte streams into slices of shared
+ * byte[]. This is used by DocumentsWriter to hold the
+ * posting list for many terms in RAM.
+ */
+
+final class ByteSliceWriter {
+
+ private byte[] slice;
+ private int upto;
+ private final ByteBlockPool pool;
+
+ int offset0;
+
+ public ByteSliceWriter(ByteBlockPool pool) {
+ this.pool = pool;
+ }
+
+ /**
+ * Set up the writer to write at address.
+ */
+ public void init(int address) {
+ slice = pool.buffers[address >> DocumentsWriter.BYTE_BLOCK_SHIFT];
+ assert slice != null;
+ upto = address & DocumentsWriter.BYTE_BLOCK_MASK;
+ offset0 = address;
+ assert upto < slice.length;
+ }
+
+ /** Write byte into byte slice stream */
+ public void writeByte(byte b) {
+ assert slice != null;
+ if (slice[upto] != 0) {
+ upto = pool.allocSlice(slice, upto);
+ slice = pool.buffer;
+ offset0 = pool.byteOffset;
+ assert slice != null;
+ }
+ slice[upto++] = b;
+ assert upto != slice.length;
+ }
+
+ public void writeBytes(final byte[] b, int offset, final int len) {
+ final int offsetEnd = offset + len;
+ while(offset < offsetEnd) {
+ if (slice[upto] != 0) {
+ // End marker
+ upto = pool.allocSlice(slice, upto);
+ slice = pool.buffer;
+ offset0 = pool.byteOffset;
+ }
+
+ slice[upto++] = b[offset++];
+ assert upto != slice.length;
+ }
+ }
+
+ public int getAddress() {
+ return upto + (offset0 & DocumentsWriter.BYTE_BLOCK_NOT_MASK);
+ }
+
+ public void writeVInt(int i) {
+ while ((i & ~0x7F) != 0) {
+ writeByte((byte)((i & 0x7f) | 0x80));
+ i >>>= 7;
+ }
+ writeByte((byte) i);
+ }
+}
Propchange: lucene/java/trunk/src/java/org/apache/lucene/index/ByteSliceWriter.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=657397&r1=657396&r2=657397&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriter.java Sat May 17 09:49:18 2008
@@ -1473,30 +1473,39 @@
final static int BYTE_BLOCK_MASK = BYTE_BLOCK_SIZE - 1;
final static int BYTE_BLOCK_NOT_MASK = ~BYTE_BLOCK_MASK;
- private ArrayList freeByteBlocks = new ArrayList();
+ private class ByteBlockAllocator extends ByteBlockPool.Allocator {
- /* Allocate another byte[] from the shared pool */
- synchronized byte[] getByteBlock(boolean trackAllocations) {
- final int size = freeByteBlocks.size();
- final byte[] b;
- if (0 == size) {
- numBytesAlloc += BYTE_BLOCK_SIZE;
- balanceRAM();
- b = new byte[BYTE_BLOCK_SIZE];
- } else
- b = (byte[]) freeByteBlocks.remove(size-1);
- if (trackAllocations)
- numBytesUsed += BYTE_BLOCK_SIZE;
- assert numBytesUsed <= numBytesAlloc;
- return b;
- }
+ ArrayList freeByteBlocks = new ArrayList();
+
+ /* Allocate another byte[] from the shared pool */
+ byte[] getByteBlock(boolean trackAllocations) {
+ synchronized(DocumentsWriter.this) {
+ final int size = freeByteBlocks.size();
+ final byte[] b;
+ if (0 == size) {
+ numBytesAlloc += BYTE_BLOCK_SIZE;
+ balanceRAM();
+ b = new byte[BYTE_BLOCK_SIZE];
+ } else
+ b = (byte[]) freeByteBlocks.remove(size-1);
+ if (trackAllocations)
+ numBytesUsed += BYTE_BLOCK_SIZE;
+ assert numBytesUsed <= numBytesAlloc;
+ return b;
+ }
+ }
- /* Return byte[]'s to the pool */
- synchronized void recycleByteBlocks(byte[][] blocks, int start, int end) {
- for(int i=start;i<end;i++)
- freeByteBlocks.add(blocks[i]);
+ /* Return byte[]'s to the pool */
+ void recycleByteBlocks(byte[][] blocks, int start, int end) {
+ synchronized(DocumentsWriter.this) {
+ for(int i=start;i<end;i++)
+ freeByteBlocks.add(blocks[i]);
+ }
+ }
}
+ ByteBlockAllocator byteBlockAllocator = new ByteBlockAllocator();
+
/* Initial chunk size of the shared char[] blocks used to
store term text */
final static int CHAR_BLOCK_SHIFT = 14;
@@ -1563,7 +1572,7 @@
" allocMB=" + toMB(numBytesAlloc) +
" vs trigger=" + toMB(freeTrigger) +
" postingsFree=" + toMB(postingsFreeCount*POSTING_NUM_BYTE) +
- " byteBlockFree=" + toMB(freeByteBlocks.size()*BYTE_BLOCK_SIZE) +
+ " byteBlockFree=" + toMB(byteBlockAllocator.freeByteBlocks.size()*BYTE_BLOCK_SIZE) +
" charBlockFree=" + toMB(freeCharBlocks.size()*CHAR_BLOCK_SIZE*CHAR_NUM_BYTE));
// When we've crossed 100% of our target Postings
@@ -1580,7 +1589,7 @@
// (freeLevel)
while(numBytesAlloc > freeLevel) {
- if (0 == freeByteBlocks.size() && 0 == freeCharBlocks.size() && 0 == postingsFreeCount) {
+ if (0 == byteBlockAllocator.freeByteBlocks.size() && 0 == freeCharBlocks.size() && 0 == postingsFreeCount) {
// Nothing else to free -- must flush now.
bufferIsFull = true;
if (infoStream != null)
@@ -1588,8 +1597,8 @@
break;
}
- if ((0 == iter % 3) && freeByteBlocks.size() > 0) {
- freeByteBlocks.remove(freeByteBlocks.size()-1);
+ if ((0 == iter % 3) && byteBlockAllocator.freeByteBlocks.size() > 0) {
+ byteBlockAllocator.freeByteBlocks.remove(byteBlockAllocator.freeByteBlocks.size()-1);
numBytesAlloc -= BYTE_BLOCK_SIZE;
}
Modified: lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriterFieldData.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriterFieldData.java?rev=657397&r1=657396&r2=657397&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriterFieldData.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriterFieldData.java Sat May 17 09:49:18 2008
@@ -64,9 +64,14 @@
float boost;
int postingsVectorsUpto;
+ final ByteSliceWriter sliceWriter;
+ final ByteSliceWriter vectorsSliceWriter;
+
public DocumentsWriterFieldData(DocumentsWriterThreadState threadState, FieldInfo fieldInfo) {
this.fieldInfo = fieldInfo;
this.threadState = threadState;
+ sliceWriter = new ByteSliceWriter(threadState.postingsPool);
+ vectorsSliceWriter = new ByteSliceWriter(threadState.vectorsPool);
}
void resetPostingArrays() {
@@ -406,15 +411,15 @@
// Now that we know doc freq for previous doc,
// write it & lastDocCode
- freqUpto = p.freqUpto & DocumentsWriter.BYTE_BLOCK_MASK;
- freq = threadState.postingsPool.buffers[p.freqUpto >> DocumentsWriter.BYTE_BLOCK_SHIFT];
+ sliceWriter.init(p.freqUpto);
+
if (1 == p.docFreq)
- writeFreqVInt(p.lastDocCode|1);
+ sliceWriter.writeVInt(p.lastDocCode|1);
else {
- writeFreqVInt(p.lastDocCode);
- writeFreqVInt(p.docFreq);
+ sliceWriter.writeVInt(p.lastDocCode);
+ sliceWriter.writeVInt(p.docFreq);
}
- p.freqUpto = freqUpto + (p.freqUpto & DocumentsWriter.BYTE_BLOCK_NOT_MASK);
+ p.freqUpto = sliceWriter.getAddress();
if (doVectors) {
vector = addNewVector();
@@ -520,155 +525,37 @@
proxCode = position;
}
- proxUpto = p.proxUpto & DocumentsWriter.BYTE_BLOCK_MASK;
- prox = threadState.postingsPool.buffers[p.proxUpto >> DocumentsWriter.BYTE_BLOCK_SHIFT];
- assert prox != null;
+ sliceWriter.init(p.proxUpto);
if (payload != null && payload.length > 0) {
- writeProxVInt((proxCode<<1)|1);
- writeProxVInt(payload.length);
- writeProxBytes(payload.data, payload.offset, payload.length);
+ sliceWriter.writeVInt((proxCode<<1)|1);
+ sliceWriter.writeVInt(payload.length);
+ sliceWriter.writeBytes(payload.data, payload.offset, payload.length);
fieldInfo.storePayloads = true;
} else
- writeProxVInt(proxCode<<1);
-
- p.proxUpto = proxUpto + (p.proxUpto & DocumentsWriter.BYTE_BLOCK_NOT_MASK);
+ sliceWriter.writeVInt(proxCode<<1);
+ p.proxUpto = sliceWriter.getAddress();
p.lastPosition = position++;
if (doVectorPositions) {
- posUpto = vector.posUpto & DocumentsWriter.BYTE_BLOCK_MASK;
- pos = threadState.vectorsPool.buffers[vector.posUpto >> DocumentsWriter.BYTE_BLOCK_SHIFT];
- writePosVInt(proxCode);
- vector.posUpto = posUpto + (vector.posUpto & DocumentsWriter.BYTE_BLOCK_NOT_MASK);
+ vectorsSliceWriter.init(vector.posUpto);
+ vectorsSliceWriter.writeVInt(proxCode);
+ vector.posUpto = vectorsSliceWriter.getAddress();
}
if (doVectorOffsets) {
- offsetUpto = vector.offsetUpto & DocumentsWriter.BYTE_BLOCK_MASK;
- offsets = threadState.vectorsPool.buffers[vector.offsetUpto >> DocumentsWriter.BYTE_BLOCK_SHIFT];
- writeOffsetVInt(offsetStartCode);
- writeOffsetVInt(offsetEnd-offsetStart);
+ vectorsSliceWriter.init(vector.offsetUpto);
+ vectorsSliceWriter.writeVInt(offsetStartCode);
+ vectorsSliceWriter.writeVInt(offsetEnd-offsetStart);
vector.lastOffset = offsetEnd;
- vector.offsetUpto = offsetUpto + (vector.offsetUpto & DocumentsWriter.BYTE_BLOCK_NOT_MASK);
+ vector.offsetUpto = vectorsSliceWriter.getAddress();
}
} catch (Throwable t) {
throw new AbortException(t, threadState.docWriter);
}
}
- /** Write vInt into freq stream of current Posting */
- public void writeFreqVInt(int i) {
- while ((i & ~0x7F) != 0) {
- writeFreqByte((byte)((i & 0x7f) | 0x80));
- i >>>= 7;
- }
- writeFreqByte((byte) i);
- }
-
- /** Write vInt into prox stream of current Posting */
- public void writeProxVInt(int i) {
- while ((i & ~0x7F) != 0) {
- writeProxByte((byte)((i & 0x7f) | 0x80));
- i >>>= 7;
- }
- writeProxByte((byte) i);
- }
-
- /** Write byte into freq stream of current Posting */
- byte[] freq;
- int freqUpto;
- public void writeFreqByte(byte b) {
- assert freq != null;
- if (freq[freqUpto] != 0) {
- freqUpto = threadState.postingsPool.allocSlice(freq, freqUpto);
- freq = threadState.postingsPool.buffer;
- p.freqUpto = threadState.postingsPool.byteOffset;
- }
- freq[freqUpto++] = b;
- }
-
- /** Write byte into prox stream of current Posting */
- byte[] prox;
- int proxUpto;
- public void writeProxByte(byte b) {
- assert prox != null;
- if (prox[proxUpto] != 0) {
- proxUpto = threadState.postingsPool.allocSlice(prox, proxUpto);
- prox = threadState.postingsPool.buffer;
- p.proxUpto = threadState.postingsPool.byteOffset;
- assert prox != null;
- }
- prox[proxUpto++] = b;
- assert proxUpto != prox.length;
- }
-
- /** Currently only used to copy a payload into the prox
- * stream. */
- public void writeProxBytes(byte[] b, int offset, int len) {
- final int offsetEnd = offset + len;
- while(offset < offsetEnd) {
- if (prox[proxUpto] != 0) {
- // End marker
- proxUpto = threadState.postingsPool.allocSlice(prox, proxUpto);
- prox = threadState.postingsPool.buffer;
- p.proxUpto = threadState.postingsPool.byteOffset;
- }
-
- prox[proxUpto++] = b[offset++];
- assert proxUpto != prox.length;
- }
- }
-
- /** Write vInt into offsets stream of current
- * PostingVector */
- public void writeOffsetVInt(int i) {
- while ((i & ~0x7F) != 0) {
- writeOffsetByte((byte)((i & 0x7f) | 0x80));
- i >>>= 7;
- }
- writeOffsetByte((byte) i);
- }
-
- byte[] offsets;
- int offsetUpto;
-
- /** Write byte into offsets stream of current
- * PostingVector */
- public void writeOffsetByte(byte b) {
- assert offsets != null;
- if (offsets[offsetUpto] != 0) {
- offsetUpto = threadState.vectorsPool.allocSlice(offsets, offsetUpto);
- offsets = threadState.vectorsPool.buffer;
- vector.offsetUpto = threadState.vectorsPool.byteOffset;
- }
- offsets[offsetUpto++] = b;
- }
-
- /** Write vInt into pos stream of current
- * PostingVector */
- public void writePosVInt(int i) {
- while ((i & ~0x7F) != 0) {
- writePosByte((byte)((i & 0x7f) | 0x80));
- i >>>= 7;
- }
- writePosByte((byte) i);
- }
-
- byte[] pos;
- int posUpto;
-
- /** Write byte into pos stream of current
- * PostingVector */
- public void writePosByte(byte b) {
- assert pos != null;
- if (pos[posUpto] != 0) {
- posUpto = threadState.vectorsPool.allocSlice(pos, posUpto);
- pos = threadState.vectorsPool.buffer;
- vector.posUpto = threadState.vectorsPool.byteOffset;
- }
- pos[posUpto++] = b;
- }
-
/** Called when postings hash is too small (> 50%
* occupied) or too large (< 20% occupied). */
void rehashPostings(final int newSize) {
Modified: lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriterThreadState.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriterThreadState.java?rev=657397&r1=657396&r2=657397&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriterThreadState.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/DocumentsWriterThreadState.java Sat May 17 09:49:18 2008
@@ -80,8 +80,8 @@
postingsFreeList = new Posting[256];
postingsFreeCount = 0;
- postingsPool = new ByteBlockPool(docWriter ,true);
- vectorsPool = new ByteBlockPool(docWriter, false);
+ postingsPool = new ByteBlockPool(docWriter.byteBlockAllocator, true);
+ vectorsPool = new ByteBlockPool(docWriter.byteBlockAllocator, false);
charPool = new CharBlockPool(docWriter);
}
Added: lucene/java/trunk/src/test/org/apache/lucene/index/TestByteSlices.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestByteSlices.java?rev=657397&view=auto
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestByteSlices.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestByteSlices.java Sat May 17 09:49:18 2008
@@ -0,0 +1,109 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Random;
+import java.util.ArrayList;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestByteSlices extends LuceneTestCase {
+
+ private static class ByteBlockAllocator extends ByteBlockPool.Allocator {
+ ArrayList freeByteBlocks = new ArrayList();
+
+ /* Allocate another byte[] from the shared pool */
+ synchronized byte[] getByteBlock(boolean trackAllocations) {
+ final int size = freeByteBlocks.size();
+ final byte[] b;
+ if (0 == size)
+ b = new byte[DocumentsWriter.BYTE_BLOCK_SIZE];
+ else
+ b = (byte[]) freeByteBlocks.remove(size-1);
+ return b;
+ }
+
+ /* Return a byte[] to the pool */
+ synchronized void recycleByteBlocks(byte[][] blocks, int start, int end) {
+ for(int i=start;i<end;i++)
+ freeByteBlocks.add(blocks[i]);
+ }
+ }
+
+ public void testBasic() throws Throwable {
+ ByteBlockPool pool = new ByteBlockPool(new ByteBlockAllocator(), false);
+
+ final int NUM_STREAM = 25;
+
+ ByteSliceWriter writer = new ByteSliceWriter(pool);
+
+ int[] starts = new int[NUM_STREAM];
+ int[] uptos = new int[NUM_STREAM];
+ int[] counters = new int[NUM_STREAM];
+
+ Random r = new Random(1);
+
+ ByteSliceReader reader = new ByteSliceReader();
+
+ for(int ti=0;ti<100;ti++) {
+
+ for(int stream=0;stream<NUM_STREAM;stream++) {
+ starts[stream] = -1;
+ counters[stream] = 0;
+ }
+
+ boolean debug = false;
+
+ for(int iter=0;iter<10000;iter++) {
+ int stream = r.nextInt(NUM_STREAM);
+ if (debug)
+ System.out.println("write stream=" + stream);
+
+ if (starts[stream] == -1) {
+ final int spot = pool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
+ starts[stream] = uptos[stream] = spot + pool.byteOffset;
+ if (debug)
+ System.out.println(" init to " + starts[stream]);
+ }
+
+ writer.init(uptos[stream]);
+ int numValue = r.nextInt(20);
+ for(int j=0;j<numValue;j++) {
+ if (debug)
+ System.out.println(" write " + (counters[stream]+j));
+ writer.writeVInt(counters[stream]+j);
+ //writer.writeVInt(ti);
+ }
+ counters[stream] += numValue;
+ uptos[stream] = writer.getAddress();
+ if (debug)
+ System.out.println(" addr now " + uptos[stream]);
+ }
+
+ for(int stream=0;stream<NUM_STREAM;stream++) {
+ if (debug)
+ System.out.println(" stream=" + stream + " count=" + counters[stream]);
+
+ if (starts[stream] != uptos[stream]) {
+ reader.init(pool, starts[stream], uptos[stream]);
+ for(int j=0;j<counters[stream];j++)
+ assertEquals(j, reader.readVInt());
+ //assertEquals(ti, reader.readVInt());
+ }
+ }
+
+ pool.reset();
+ }
+ }
+}
Propchange: lucene/java/trunk/src/test/org/apache/lucene/index/TestByteSlices.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=657397&r1=657396&r2=657397&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestStressIndexing2.java Sat May 17 09:49:18 2008
@@ -112,45 +112,48 @@
// everything.
public Map indexRandom(int nThreads, int iterations, int range, Directory dir) throws IOException, InterruptedException {
- IndexWriter w = new MockIndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
- w.setUseCompoundFile(false);
- /***
- w.setMaxMergeDocs(Integer.MAX_VALUE);
- w.setMaxFieldLength(10000);
- w.setRAMBufferSizeMB(1);
- w.setMergeFactor(10);
- ***/
-
- // force many merges
- w.setMergeFactor(mergeFactor);
- w.setRAMBufferSizeMB(.1);
- w.setMaxBufferedDocs(maxBufferedDocs);
-
- threads = new IndexingThread[nThreads];
- for (int i=0; i<threads.length; i++) {
- IndexingThread th = new IndexingThread();
- th.w = w;
- th.base = 1000000*i;
- th.range = range;
- th.iterations = iterations;
- threads[i] = th;
- }
-
- for (int i=0; i<threads.length; i++) {
- threads[i].start();
- }
- for (int i=0; i<threads.length; i++) {
- threads[i].join();
- }
-
- // w.optimize();
- w.close();
-
Map docs = new HashMap();
- for (int i=0; i<threads.length; i++) {
- IndexingThread th = threads[i];
- synchronized(th) {
- docs.putAll(th.docs);
+ for(int iter=0;iter<3;iter++) {
+ IndexWriter w = new MockIndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
+ w.setUseCompoundFile(false);
+
+ /***
+ w.setMaxMergeDocs(Integer.MAX_VALUE);
+ w.setMaxFieldLength(10000);
+ w.setRAMBufferSizeMB(1);
+ w.setMergeFactor(10);
+ ***/
+
+ // force many merges
+ w.setMergeFactor(mergeFactor);
+ w.setRAMBufferSizeMB(.1);
+ w.setMaxBufferedDocs(maxBufferedDocs);
+
+ threads = new IndexingThread[nThreads];
+ for (int i=0; i<threads.length; i++) {
+ IndexingThread th = new IndexingThread();
+ th.w = w;
+ th.base = 1000000*i;
+ th.range = range;
+ th.iterations = iterations;
+ threads[i] = th;
+ }
+
+ for (int i=0; i<threads.length; i++) {
+ threads[i].start();
+ }
+ for (int i=0; i<threads.length; i++) {
+ threads[i].join();
+ }
+
+ // w.optimize();
+ w.close();
+
+ for (int i=0; i<threads.length; i++) {
+ IndexingThread th = threads[i];
+ synchronized(th) {
+ docs.putAll(th.docs);
+ }
}
}
@@ -515,16 +518,16 @@
switch (nextInt(4)) {
case 0:
- fields.add(new Field("f0", getString(1), Field.Store.YES, Field.Index.NO_NORMS, tvVal));
+ fields.add(new Field("f" + nextInt(100), getString(1), Field.Store.YES, Field.Index.NO_NORMS, tvVal));
break;
case 1:
- fields.add(new Field("f1", getString(0), Field.Store.NO, Field.Index.TOKENIZED, tvVal));
+ fields.add(new Field("f" + nextInt(100), getString(0), Field.Store.NO, Field.Index.TOKENIZED, tvVal));
break;
case 2:
- fields.add(new Field("f2", getString(0), Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
+ fields.add(new Field("f" + nextInt(100), getString(0), Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
break;
case 3:
- fields.add(new Field("f3", getString(bigFieldSize), Field.Store.YES, Field.Index.TOKENIZED, tvVal));
+ fields.add(new Field("f" + nextInt(100), getString(bigFieldSize), Field.Store.YES, Field.Index.TOKENIZED, tvVal));
break;
}
}