You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2011/07/08 23:02:25 UTC
svn commit: r1144510 [3/5] - in /lucene/dev/branches/solr2452: ./
dev-tools/idea/lucene/contrib/ lucene/
lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/
lucene/contrib/memory/src/java/org/apache/lucene/index/memory/ lucene/co...
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java Fri Jul 8 21:02:20 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.index.codecs.Bl
import org.apache.lucene.index.codecs.PostingsReaderBase;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
@@ -51,15 +52,15 @@ public class StandardPostingsReader exte
//private String segment;
- public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize, int codecId) throws IOException {
+ public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext context, int codecId) throws IOException {
freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.FREQ_EXTENSION),
- readBufferSize);
+ context);
//this.segment = segmentInfo.name;
if (segmentInfo.getHasProx()) {
boolean success = false;
try {
proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.PROX_EXTENSION),
- readBufferSize);
+ context);
success = true;
} finally {
if (!success) {
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java Fri Jul 8 21:02:20 2011
@@ -92,14 +92,14 @@ public final class StandardPostingsWrite
this.skipMinimum = skipInterval; /* set to the same for now */
//this.segment = state.segmentName;
String fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.FREQ_EXTENSION);
- freqOut = state.directory.createOutput(fileName);
+ freqOut = state.directory.createOutput(fileName, state.context);
boolean success = false;
try {
if (state.fieldInfos.hasProx()) {
// At least one field does not omit TF, so create the
// prox file
fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.PROX_EXTENSION);
- proxOut = state.directory.createOutput(fileName);
+ proxOut = state.directory.createOutput(fileName, state.context);
} else {
// Every field omits TF so we will write no prox file
proxOut = null;
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Bytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Bytes.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Bytes.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Bytes.java Fri Jul 8 21:02:20 2011
@@ -28,6 +28,7 @@ import org.apache.lucene.index.values.In
import org.apache.lucene.index.values.IndexDocValues.Source;
import org.apache.lucene.index.values.IndexDocValues.SourceEnum;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.AttributeSource;
@@ -100,12 +101,13 @@ public final class Bytes {
* {@link Writer}. A call to {@link Writer#finish(int)} will release
* all internally used resources and frees the memeory tracking
* reference.
+ * @param context
* @return a new {@link Writer} instance
* @throws IOException
* if the files for the writer can not be created.
*/
public static Writer getWriter(Directory dir, String id, Mode mode,
- Comparator<BytesRef> comp, boolean fixedSize, AtomicLong bytesUsed)
+ Comparator<BytesRef> comp, boolean fixedSize, AtomicLong bytesUsed, IOContext context)
throws IOException {
// TODO -- i shouldn't have to specify fixed? can
// track itself & do the write thing at write time?
@@ -115,19 +117,19 @@ public final class Bytes {
if (fixedSize) {
if (mode == Mode.STRAIGHT) {
- return new FixedStraightBytesImpl.Writer(dir, id, bytesUsed);
+ return new FixedStraightBytesImpl.Writer(dir, id, bytesUsed, context);
} else if (mode == Mode.DEREF) {
- return new FixedDerefBytesImpl.Writer(dir, id, bytesUsed);
+ return new FixedDerefBytesImpl.Writer(dir, id, bytesUsed, context);
} else if (mode == Mode.SORTED) {
- return new FixedSortedBytesImpl.Writer(dir, id, comp, bytesUsed);
+ return new FixedSortedBytesImpl.Writer(dir, id, comp, bytesUsed, context);
}
} else {
if (mode == Mode.STRAIGHT) {
- return new VarStraightBytesImpl.Writer(dir, id, bytesUsed);
+ return new VarStraightBytesImpl.Writer(dir, id, bytesUsed, context);
} else if (mode == Mode.DEREF) {
- return new VarDerefBytesImpl.Writer(dir, id, bytesUsed);
+ return new VarDerefBytesImpl.Writer(dir, id, bytesUsed, context);
} else if (mode == Mode.SORTED) {
- return new VarSortedBytesImpl.Writer(dir, id, comp, bytesUsed);
+ return new VarSortedBytesImpl.Writer(dir, id, comp, bytesUsed, context);
}
}
@@ -157,23 +159,24 @@ public final class Bytes {
* if an {@link IOException} occurs
*/
public static IndexDocValues getValues(Directory dir, String id, Mode mode,
- boolean fixedSize, int maxDoc, Comparator<BytesRef> sortComparator) throws IOException {
+ boolean fixedSize, int maxDoc, Comparator<BytesRef> sortComparator, IOContext context) throws IOException {
+
// TODO -- I can peek @ header to determing fixed/mode?
if (fixedSize) {
if (mode == Mode.STRAIGHT) {
- return new FixedStraightBytesImpl.Reader(dir, id, maxDoc);
+ return new FixedStraightBytesImpl.Reader(dir, id, maxDoc, context);
} else if (mode == Mode.DEREF) {
- return new FixedDerefBytesImpl.Reader(dir, id, maxDoc);
+ return new FixedDerefBytesImpl.Reader(dir, id, maxDoc, context);
} else if (mode == Mode.SORTED) {
- return new FixedSortedBytesImpl.Reader(dir, id, maxDoc);
+ return new FixedSortedBytesImpl.Reader(dir, id, maxDoc, context);
}
} else {
if (mode == Mode.STRAIGHT) {
- return new VarStraightBytesImpl.Reader(dir, id, maxDoc);
+ return new VarStraightBytesImpl.Reader(dir, id, maxDoc, context);
} else if (mode == Mode.DEREF) {
- return new VarDerefBytesImpl.Reader(dir, id, maxDoc);
+ return new VarDerefBytesImpl.Reader(dir, id, maxDoc, context);
} else if (mode == Mode.SORTED) {
- return new VarSortedBytesImpl.Reader(dir, id, maxDoc, sortComparator);
+ return new VarSortedBytesImpl.Reader(dir, id, maxDoc, sortComparator, context);
}
}
@@ -343,15 +346,16 @@ public final class Bytes {
private final Directory dir;
private final String codecName;
private final int version;
+ private final IOContext context;
protected BytesWriterBase(Directory dir, String id, String codecName,
- int version,
- AtomicLong bytesUsed) throws IOException {
+ int version, AtomicLong bytesUsed, IOContext context) throws IOException {
super(bytesUsed);
this.id = id;
this.dir = dir;
this.codecName = codecName;
this.version = version;
+ this.context = context;
}
protected IndexOutput getDataOut() throws IOException {
@@ -359,7 +363,7 @@ public final class Bytes {
boolean success = false;
try {
datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "",
- DATA_EXTENSION));
+ DATA_EXTENSION), context);
CodecUtil.writeHeader(datOut, codecName, version);
success = true;
} finally {
@@ -376,7 +380,7 @@ public final class Bytes {
try {
if (idxOut == null) {
idxOut = dir.createOutput(IndexFileNames.segmentFileName(id, "",
- INDEX_EXTENSION));
+ INDEX_EXTENSION), context);
CodecUtil.writeHeader(idxOut, codecName, version);
}
success = true;
@@ -439,16 +443,16 @@ public final class Bytes {
protected final String id;
protected BytesReaderBase(Directory dir, String id, String codecName,
- int maxVersion, boolean doIndex) throws IOException {
+ int maxVersion, boolean doIndex, IOContext context) throws IOException {
this.id = id;
datIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
- Writer.DATA_EXTENSION));
+ Writer.DATA_EXTENSION), context);
boolean success = false;
try {
version = CodecUtil.checkHeader(datIn, codecName, maxVersion, maxVersion);
if (doIndex) {
idxIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
- Writer.INDEX_EXTENSION));
+ Writer.INDEX_EXTENSION), context);
final int version2 = CodecUtil.checkHeader(idxIn, codecName,
maxVersion, maxVersion);
assert version == version2;
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java Fri Jul 8 21:02:20 2011
@@ -24,6 +24,7 @@ import org.apache.lucene.index.values.By
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
import org.apache.lucene.index.values.Bytes.BytesWriterBase;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
@@ -54,15 +55,15 @@ class FixedDerefBytesImpl {
private int size = -1;
private int[] docToID;
private final BytesRefHash hash;
- public Writer(Directory dir, String id, AtomicLong bytesUsed)
+ public Writer(Directory dir, String id, AtomicLong bytesUsed, IOContext context)
throws IOException {
this(dir, id, new DirectTrackingAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed),
- bytesUsed);
+ bytesUsed, context);
}
public Writer(Directory dir, String id, Allocator allocator,
- AtomicLong bytesUsed) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed);
+ AtomicLong bytesUsed, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
hash = new BytesRefHash(new ByteBlockPool(allocator),
BytesRefHash.DEFAULT_CAPACITY, new TrackingDirectBytesStartArray(
BytesRefHash.DEFAULT_CAPACITY, bytesUsed));
@@ -144,8 +145,8 @@ class FixedDerefBytesImpl {
public static class Reader extends BytesReaderBase {
private final int size;
- Reader(Directory dir, String id, int maxDoc) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_START, true);
+ Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_START, true, context);
size = datIn.readInt();
}
@@ -220,8 +221,10 @@ class FixedDerefBytesImpl {
idxIn.readInt();// read valueCount
idx = PackedInts.getReaderIterator(idxIn);
fp = datIn.getFilePointer();
- bytesRef.grow(this.size);
- bytesRef.length = this.size;
+ if (size > 0) {
+ bytesRef.grow(this.size);
+ bytesRef.length = this.size;
+ }
bytesRef.offset = 0;
valueCount = idx.size();
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java Fri Jul 8 21:02:20 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.index.values.By
import org.apache.lucene.index.values.Bytes.BytesWriterBase;
import org.apache.lucene.index.values.FixedDerefBytesImpl.Reader.DerefBytesEnum;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
@@ -61,14 +62,14 @@ class FixedSortedBytesImpl {
private final BytesRefHash hash;
public Writer(Directory dir, String id, Comparator<BytesRef> comp,
- AtomicLong bytesUsed) throws IOException {
+ AtomicLong bytesUsed, IOContext context) throws IOException {
this(dir, id, comp, new DirectTrackingAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed),
- bytesUsed);
+ bytesUsed, context);
}
public Writer(Directory dir, String id, Comparator<BytesRef> comp,
- Allocator allocator, AtomicLong bytesUsed) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed);
+ Allocator allocator, AtomicLong bytesUsed, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
ByteBlockPool pool = new ByteBlockPool(allocator);
hash = new BytesRefHash(pool, BytesRefHash.DEFAULT_CAPACITY,
new TrackingDirectBytesStartArray(BytesRefHash.DEFAULT_CAPACITY,
@@ -169,8 +170,8 @@ class FixedSortedBytesImpl {
public static class Reader extends BytesReaderBase {
private final int size;
- public Reader(Directory dir, String id, int maxDoc) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_START, true);
+ public Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_START, true, context);
size = datIn.readInt();
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java Fri Jul 8 21:02:20 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.index.values.By
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
import org.apache.lucene.index.values.Bytes.BytesWriterBase;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.AttributeSource;
@@ -55,8 +56,8 @@ class FixedStraightBytesImpl {
private final int byteBlockSize;
private IndexOutput datOut;
- public Writer(Directory dir, String id, AtomicLong bytesUsed) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed);
+ public Writer(Directory dir, String id, AtomicLong bytesUsed, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
pool = new ByteBlockPool(new DirectTrackingAllocator(bytesUsed));
byteBlockSize = BYTE_BLOCK_SIZE;
}
@@ -204,8 +205,8 @@ class FixedStraightBytesImpl {
private final int size;
private final int maxDoc;
- Reader(Directory dir, String id, int maxDoc) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_START, false);
+ Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_START, false, context);
size = datIn.readInt();
this.maxDoc = maxDoc;
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Floats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Floats.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Floats.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Floats.java Fri Jul 8 21:02:20 2011
@@ -23,6 +23,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.values.IndexDocValues.Source;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
@@ -49,21 +50,21 @@ public class Floats {
private static final byte[] DEFAULTS = new byte[] {0,0,0,0,0,0,0,0};
public static Writer getWriter(Directory dir, String id, int precisionBytes,
- AtomicLong bytesUsed) throws IOException {
+ AtomicLong bytesUsed, IOContext context) throws IOException {
if (precisionBytes != 4 && precisionBytes != 8) {
throw new IllegalArgumentException("precisionBytes must be 4 or 8; got "
+ precisionBytes);
}
if (precisionBytes == 4) {
- return new Float4Writer(dir, id, bytesUsed);
+ return new Float4Writer(dir, id, bytesUsed, context);
} else {
- return new Float8Writer(dir, id, bytesUsed);
+ return new Float8Writer(dir, id, bytesUsed, context);
}
}
- public static IndexDocValues getValues(Directory dir, String id, int maxDoc)
+ public static IndexDocValues getValues(Directory dir, String id, int maxDoc, IOContext context)
throws IOException {
- return new FloatsReader(dir, id, maxDoc);
+ return new FloatsReader(dir, id, maxDoc, context);
}
abstract static class FloatsWriter extends Writer {
@@ -73,13 +74,15 @@ public class Floats {
protected IndexOutput datOut;
private final byte precision;
private final Directory dir;
+ private final IOContext context;
protected FloatsWriter(Directory dir, String id, int precision,
- AtomicLong bytesUsed) throws IOException {
+ AtomicLong bytesUsed, IOContext context) throws IOException {
super(bytesUsed);
this.id = id;
this.precision = (byte) precision;
this.dir = dir;
+ this.context = context;
}
@@ -90,7 +93,7 @@ public class Floats {
final void initDataOut() throws IOException {
assert datOut == null;
datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "",
- Writer.DATA_EXTENSION));
+ Writer.DATA_EXTENSION), context);
boolean success = false;
try {
CodecUtil.writeHeader(datOut, CODEC_NAME, VERSION_CURRENT);
@@ -158,9 +161,9 @@ public class Floats {
// Writes 4 bytes (float) per value
static final class Float4Writer extends FloatsWriter {
private int[] values;
- protected Float4Writer(Directory dir, String id, AtomicLong bytesUsed)
+ protected Float4Writer(Directory dir, String id, AtomicLong bytesUsed, IOContext context)
throws IOException {
- super(dir, id, 4, bytesUsed);
+ super(dir, id, 4, bytesUsed, context);
values = new int[1];
bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT);
}
@@ -221,9 +224,9 @@ public class Floats {
// Writes 8 bytes (double) per value
static final class Float8Writer extends FloatsWriter {
private long[] values;
- protected Float8Writer(Directory dir, String id, AtomicLong bytesUsed)
+ protected Float8Writer(Directory dir, String id, AtomicLong bytesUsed, IOContext context)
throws IOException {
- super(dir, id, 8, bytesUsed);
+ super(dir, id, 8, bytesUsed, context);
values = new long[1];
bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_LONG);
}
@@ -288,10 +291,10 @@ public class Floats {
// TODO(simonw) is ByteBuffer the way to go here?
private final int maxDoc;
- protected FloatsReader(Directory dir, String id, int maxDoc)
+ protected FloatsReader(Directory dir, String id, int maxDoc, IOContext context)
throws IOException {
datIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
- Writer.DATA_EXTENSION));
+ Writer.DATA_EXTENSION), context);
CodecUtil.checkHeader(datIn, CODEC_NAME, VERSION_START, VERSION_START);
precisionBytes = datIn.readByte();
assert precisionBytes == 4 || precisionBytes == 8;
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Ints.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Ints.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Ints.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Ints.java Fri Jul 8 21:02:20 2011
@@ -23,6 +23,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.index.values.IntsImpl.IntsReader;
import org.apache.lucene.index.values.IntsImpl.IntsWriter;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
/**
* @lucene.experimental
@@ -33,11 +34,13 @@ public class Ints {
private Ints() {
}
- public static Writer getWriter(Directory dir, String id, AtomicLong bytesUsed, ValueType type) throws IOException {
- return new IntsWriter(dir, id, bytesUsed, type);
+ public static Writer getWriter(Directory dir, String id,
+ AtomicLong bytesUsed, ValueType type, IOContext context) throws IOException {
+ return new IntsWriter(dir, id, bytesUsed, type, context);
}
- public static IndexDocValues getValues(Directory dir, String id, int numDocs) throws IOException {
- return new IntsReader(dir, id, numDocs);
+ public static IndexDocValues getValues(Directory dir, String id,
+ int numDocs, IOContext context) throws IOException {
+ return new IntsReader(dir, id, numDocs, context);
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java Fri Jul 8 21:02:20 2011
@@ -29,6 +29,7 @@ import org.apache.lucene.index.values.In
import org.apache.lucene.index.values.IndexDocValuesArray.LongValues;
import org.apache.lucene.index.values.IndexDocValuesArray.ShortValues;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.AttributeSource;
@@ -67,11 +68,13 @@ class IntsImpl {
private final byte typeOrd;
private IndexOutput datOut;
private boolean merging;
+ private final IOContext context;
protected IntsWriter(Directory dir, String id, AtomicLong bytesUsed,
- ValueType valueType) throws IOException {
+ ValueType valueType, IOContext context) throws IOException {
super(bytesUsed);
+ this.context = context;
this.dir = dir;
this.id = id;
switch (valueType) {
@@ -122,7 +125,7 @@ class IntsImpl {
boolean success = false;
try {
datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "",
- DATA_EXTENSION));
+ DATA_EXTENSION), context);
CodecUtil.writeHeader(datOut, CODEC_NAME, VERSION_CURRENT);
datOut.writeByte(typeOrd);
success = true;
@@ -273,9 +276,9 @@ class IntsImpl {
private final byte type;
private final int numDocs;
- protected IntsReader(Directory dir, String id, int numDocs) throws IOException {
+ protected IntsReader(Directory dir, String id, int numDocs, IOContext context) throws IOException {
datIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
- Writer.DATA_EXTENSION));
+ Writer.DATA_EXTENSION), context);
this.numDocs = numDocs;
boolean success = false;
try {
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java Fri Jul 8 21:02:20 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.index.values.By
import org.apache.lucene.index.values.FixedDerefBytesImpl.Reader.DerefBytesEnum;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
@@ -117,15 +118,15 @@ class VarDerefBytesImpl {
bytesUsed);
private final BytesRefHash hash;
- public Writer(Directory dir, String id, AtomicLong bytesUsed)
+ public Writer(Directory dir, String id, AtomicLong bytesUsed, IOContext context)
throws IOException {
this(dir, id, new DirectTrackingAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed),
- bytesUsed);
+ bytesUsed, context);
}
public Writer(Directory dir, String id, Allocator allocator,
- AtomicLong bytesUsed) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed);
+ AtomicLong bytesUsed, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
hash = new BytesRefHash(new ByteBlockPool(allocator), 16, array);
docToAddress = new int[1];
bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT);
@@ -220,8 +221,8 @@ class VarDerefBytesImpl {
public static class Reader extends BytesReaderBase {
- Reader(Directory dir, String id, int maxDoc) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_START, true);
+ Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_START, true, context);
}
@Override
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java Fri Jul 8 21:02:20 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.index.values.By
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
import org.apache.lucene.index.values.Bytes.BytesWriterBase;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
@@ -61,14 +62,14 @@ class VarSortedBytesImpl {
private final BytesRefHash hash;
public Writer(Directory dir, String id, Comparator<BytesRef> comp,
- AtomicLong bytesUsed) throws IOException {
+ AtomicLong bytesUsed, IOContext context) throws IOException {
this(dir, id, comp, new DirectTrackingAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed),
- bytesUsed);
+ bytesUsed, context);
}
public Writer(Directory dir, String id, Comparator<BytesRef> comp,
- Allocator allocator, AtomicLong bytesUsed) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed);
+ Allocator allocator, AtomicLong bytesUsed, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
this.hash = new BytesRefHash(new ByteBlockPool(allocator),
BytesRefHash.DEFAULT_CAPACITY, new TrackingDirectBytesStartArray(
BytesRefHash.DEFAULT_CAPACITY, bytesUsed));
@@ -168,8 +169,9 @@ class VarSortedBytesImpl {
public static class Reader extends BytesReaderBase {
private final Comparator<BytesRef> defaultComp;
- Reader(Directory dir, String id, int maxDoc, Comparator<BytesRef> comparator) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_START, true);
+
+ Reader(Directory dir, String id, int maxDoc, Comparator<BytesRef> comparator, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_START, true, context);
this.defaultComp = comparator;
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java Fri Jul 8 21:02:20 2011
@@ -24,6 +24,7 @@ import org.apache.lucene.index.values.By
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
import org.apache.lucene.index.values.Bytes.BytesWriterBase;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
@@ -56,9 +57,9 @@ class VarStraightBytesImpl {
private final ByteBlockPool pool;
private IndexOutput datOut;
private boolean merge = false;
- public Writer(Directory dir, String id, AtomicLong bytesUsed)
+ public Writer(Directory dir, String id, AtomicLong bytesUsed, IOContext context)
throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
pool = new ByteBlockPool(new DirectTrackingAllocator(bytesUsed));
docToAddress = new long[1];
pool.nextBuffer(); // init
@@ -215,8 +216,8 @@ class VarStraightBytesImpl {
public static class Reader extends BytesReaderBase {
private final int maxDoc;
- Reader(Directory dir, String id, int maxDoc) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_START, true);
+ Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_START, true, context);
this.maxDoc = maxDoc;
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Writer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Writer.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Writer.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Writer.java Fri Jul 8 21:02:20 2011
@@ -22,6 +22,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -192,7 +193,7 @@ public abstract class Writer extends Doc
* @throws IOException
*/
public static Writer create(ValueType type, String id, Directory directory,
- Comparator<BytesRef> comp, AtomicLong bytesUsed) throws IOException {
+ Comparator<BytesRef> comp, AtomicLong bytesUsed, IOContext context) throws IOException {
if (comp == null) {
comp = BytesRef.getUTF8SortedAsUnicodeComparator();
}
@@ -202,29 +203,29 @@ public abstract class Writer extends Doc
case FIXED_INTS_64:
case FIXED_INTS_8:
case VAR_INTS:
- return Ints.getWriter(directory, id, bytesUsed, type);
+ return Ints.getWriter(directory, id, bytesUsed, type, context);
case FLOAT_32:
- return Floats.getWriter(directory, id, 4, bytesUsed);
+ return Floats.getWriter(directory, id, 4, bytesUsed, context);
case FLOAT_64:
- return Floats.getWriter(directory, id, 8, bytesUsed);
+ return Floats.getWriter(directory, id, 8, bytesUsed, context);
case BYTES_FIXED_STRAIGHT:
return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, comp, true,
- bytesUsed);
+ bytesUsed, context);
case BYTES_FIXED_DEREF:
return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, comp, true,
- bytesUsed);
+ bytesUsed, context);
case BYTES_FIXED_SORTED:
return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, comp, true,
- bytesUsed);
+ bytesUsed, context);
case BYTES_VAR_STRAIGHT:
return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, comp, false,
- bytesUsed);
+ bytesUsed, context);
case BYTES_VAR_DEREF:
return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, comp, false,
- bytesUsed);
+ bytesUsed, context);
case BYTES_VAR_SORTED:
return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, comp, false,
- bytesUsed);
+ bytesUsed, context);
default:
throw new IllegalArgumentException("Unknown Values: " + type);
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java Fri Jul 8 21:02:20 2011
@@ -22,8 +22,19 @@ import java.io.IOException;
/** Base implementation class for buffered {@link IndexInput}. */
public abstract class BufferedIndexInput extends IndexInput {
- /** Default buffer size */
+ /** Default buffer size set to 1024*/
public static final int BUFFER_SIZE = 1024;
+
+ // The normal read buffer size defaults to 1024, but
+ // increasing this during merging seems to yield
+ // performance gains. However we don't want to increase
+ // it too much because there are quite a few
+ // BufferedIndexInputs created during merging. See
+ // LUCENE-888 for details.
+ /**
+ * A buffer size for merges set to 4096
+ */
+ public static final int MERGE_BUFFER_SIZE = 4096;
private int bufferSize = BUFFER_SIZE;
@@ -41,6 +52,10 @@ public abstract class BufferedIndexInput
}
public BufferedIndexInput() {}
+
+ public BufferedIndexInput(IOContext context) {
+ this(bufferSize(context));
+ }
/** Inits BufferedIndexInput with a specific bufferSize */
public BufferedIndexInput(int bufferSize) {
@@ -300,4 +315,21 @@ public abstract class BufferedIndexInput
}
}
+ /**
+ * Returns default buffer sizes for the given {@link IOContext}
+ */
+ public static int bufferSize(IOContext context) {
+ switch (context.context) {
+ case DEFAULT:
+ case FLUSH:
+ case READ:
+ return BUFFER_SIZE;
+ case MERGE:
+ return MERGE_BUFFER_SIZE;
+ default:
+ assert false : "unknown IOContext " + context.context;
+ return BUFFER_SIZE;
+ }
+ }
+
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java Fri Jul 8 21:02:20 2011
@@ -48,7 +48,7 @@ public abstract class CompoundFileDirect
private final Directory directory;
private final String fileName;
- private final int readBufferSize;
+ protected final int readBufferSize;
private Map<String,FileEntry> entries;
private boolean openForWrite;
private static final Map<String,FileEntry> SENTINEL = Collections.emptyMap();
@@ -59,11 +59,11 @@ public abstract class CompoundFileDirect
* <p>
* NOTE: subclasses must call {@link #initForRead(Map)} before the directory can be used.
*/
- public CompoundFileDirectory(Directory directory, String fileName, int readBufferSize) throws IOException {
+ public CompoundFileDirectory(Directory directory, String fileName, IOContext context) throws IOException {
this.directory = directory;
this.fileName = fileName;
- this.readBufferSize = readBufferSize;
+ this.readBufferSize = BufferedIndexInput.bufferSize(context);
this.isOpen = false;
}
@@ -91,7 +91,7 @@ public abstract class CompoundFileDirect
IndexInput input = null;
try {
input = dir.openInput(IndexFileNames.segmentFileName(IndexFileNames.stripExtension(name), "",
- IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+ IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION), IOContext.READONCE);
final int readInt = input.readInt(); // unused right now
assert readInt == CompoundFileWriter.ENTRY_FORMAT_CURRENT;
final int numEntries = input.readVInt();
@@ -189,13 +189,7 @@ public abstract class CompoundFileDirect
}
@Override
- public synchronized IndexInput openInput(String id) throws IOException {
- // Default to readBufferSize passed in when we were opened
- return openInput(id, readBufferSize);
- }
-
- @Override
- public synchronized IndexInput openInput(String id, int readBufferSize) throws IOException {
+ public synchronized IndexInput openInput(String id, IOContext context) throws IOException {
ensureOpen();
assert !openForWrite;
id = IndexFileNames.stripSegmentName(id);
@@ -273,9 +267,9 @@ public abstract class CompoundFileDirect
}
@Override
- public IndexOutput createOutput(String name) throws IOException {
+ public IndexOutput createOutput(String name, IOContext context) throws IOException {
ensureOpen();
- return writer.createOutput(name);
+ return writer.createOutput(name, context);
}
@Override
@@ -291,18 +285,18 @@ public abstract class CompoundFileDirect
}
@Override
- public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+ public CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException {
FileEntry fileEntry = this.entries.get(IndexFileNames.stripSegmentName(name));
if (fileEntry == null) {
throw new FileNotFoundException("file " + name + " does not exists in this CFS");
}
- return new NestedCompoundFileDirectory(name, bufferSize, fileEntry.offset, fileEntry.length);
+ return new NestedCompoundFileDirectory(name, context, fileEntry.offset, fileEntry.length);
}
/** Not implemented
* @throws UnsupportedOperationException */
@Override
- public CompoundFileDirectory createCompoundOutput(String name)
+ public CompoundFileDirectory createCompoundOutput(String name, IOContext context)
throws IOException {
throw new UnsupportedOperationException("can not create nested CFS, create seperately and use Directory.copy instead");
}
@@ -312,14 +306,14 @@ public abstract class CompoundFileDirect
private final long cfsOffset;
private final long cfsLength;
- public NestedCompoundFileDirectory(String fileName, int readBufferSize, long offset, long length)
+ public NestedCompoundFileDirectory(String fileName, IOContext context, long offset, long length)
throws IOException {
- super(directory, fileName, readBufferSize);
+ super(directory, fileName, context);
this.cfsOffset = offset;
this.cfsLength = length;
IndexInput input = null;
try {
- input = CompoundFileDirectory.this.openInput(fileName, 128);
+ input = CompoundFileDirectory.this.openInput(fileName, IOContext.READONCE);
initForRead(CompoundFileDirectory.readEntries(input,
CompoundFileDirectory.this, fileName));
} finally {
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java Fri Jul 8 21:02:20 2011
@@ -136,7 +136,7 @@ final class CompoundFileWriter implement
IOException priorException = null;
IndexOutput entryTableOut = null;
try {
- initDataOut();
+ initDataOut(IOContext.DEFAULT);
if (!pendingEntries.isEmpty() || outputTaken.get()) {
throw new IllegalStateException("CFS has pending open files");
}
@@ -151,7 +151,7 @@ final class CompoundFileWriter implement
IOUtils.closeSafely(priorException, dataOut);
}
try {
- entryTableOut = directory.createOutput(entryTableName);
+ entryTableOut = directory.createOutput(entryTableName, IOContext.DEFAULT);
writeEntryTable(entries.values(), entryTableOut);
} catch (IOException e) {
priorException = e;
@@ -180,7 +180,7 @@ final class CompoundFileWriter implement
*/
private final long copyFileEntry(IndexOutput dataOut, FileEntry fileEntry)
throws IOException, MergeAbortedException {
- final IndexInput is = fileEntry.dir.openInput(fileEntry.file);
+ final IndexInput is = fileEntry.dir.openInput(fileEntry.file, IOContext.READONCE);
try {
final long startPtr = dataOut.getFilePointer();
final long length = fileEntry.length;
@@ -212,7 +212,7 @@ final class CompoundFileWriter implement
}
}
- IndexOutput createOutput(String name) throws IOException {
+ IndexOutput createOutput(String name, IOContext context) throws IOException {
ensureOpen();
boolean success = false;
try {
@@ -225,7 +225,7 @@ final class CompoundFileWriter implement
entries.put(name, entry);
final DirectCFSIndexOutput out;
if (outputTaken.compareAndSet(false, true)) {
- initDataOut();
+ initDataOut(context);
success = true;
out = new DirectCFSIndexOutput(dataOut, entry, false);
} else {
@@ -233,7 +233,7 @@ final class CompoundFileWriter implement
if (directory.fileExists(name)) {
throw new IOException("File already exists");
}
- out = new DirectCFSIndexOutput(directory.createOutput(name), entry,
+ out = new DirectCFSIndexOutput(directory.createOutput(name, context), entry,
true);
}
success = true;
@@ -249,11 +249,11 @@ final class CompoundFileWriter implement
outputTaken.compareAndSet(true, false);
}
- private synchronized final void initDataOut() throws IOException {
+ private synchronized final void initDataOut(IOContext context) throws IOException {
if (dataOut == null) {
boolean success = false;
try {
- dataOut = directory.createOutput(dataFileName);
+ dataOut = directory.createOutput(dataFileName, context);
dataOut.writeVInt(FORMAT_CURRENT);
success = true;
} finally {
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/DefaultCompoundFileDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/DefaultCompoundFileDirectory.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/DefaultCompoundFileDirectory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/DefaultCompoundFileDirectory.java Fri Jul 8 21:02:20 2011
@@ -31,11 +31,11 @@ import org.apache.lucene.util.IOUtils;
public class DefaultCompoundFileDirectory extends CompoundFileDirectory {
protected IndexInput stream;
- public DefaultCompoundFileDirectory(Directory directory, String fileName, int readBufferSize, boolean writeable) throws IOException {
- super(directory, fileName, readBufferSize);
+ public DefaultCompoundFileDirectory(Directory directory, String fileName, IOContext context, boolean writeable) throws IOException {
+ super(directory, fileName, context);
if (!writeable) {
try {
- stream = directory.openInput(fileName, readBufferSize);
+ stream = directory.openInput(fileName, context);
initForRead(CompoundFileDirectory.readEntries(stream, directory, fileName));
} catch (IOException e) {
IOUtils.closeSafely(e, stream);
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/Directory.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/Directory.java Fri Jul 8 21:02:20 2011
@@ -52,7 +52,7 @@ public abstract class Directory implemen
* Returns an array of strings, one for each file in the directory.
*
* @throws NoSuchDirectoryException if the directory is not prepared for any
- * write operations (such as {@link #createOutput(String)}).
+ * write operations (such as {@link #createOutput(String, IOContext)}).
* @throws IOException in case of other IO errors
*/
public abstract String[] listAll() throws IOException;
@@ -87,7 +87,7 @@ public abstract class Directory implemen
/** Creates a new, empty file in the directory with the given name.
Returns a stream writing this file. */
- public abstract IndexOutput createOutput(String name)
+ public abstract IndexOutput createOutput(String name, IOContext context)
throws IOException;
/**
@@ -103,10 +103,6 @@ public abstract class Directory implemen
*/
public abstract void sync(Collection<String> names) throws IOException;
- /** Returns a stream reading an existing file. */
- public abstract IndexInput openInput(String name)
- throws IOException;
-
/** Returns a stream reading an existing file, with the
* specified read buffer size. The particular Directory
* implementation may ignore the buffer size. Currently
@@ -114,9 +110,7 @@ public abstract class Directory implemen
* parameter are {@link FSDirectory} and {@link
* CompoundFileDirectory}.
*/
- public IndexInput openInput(String name, int bufferSize) throws IOException {
- return openInput(name);
- }
+ public abstract IndexInput openInput(String name, IOContext context) throws IOException;
/**
* Returns a {@link CompoundFileDirectory} capable of
@@ -126,8 +120,8 @@ public abstract class Directory implemen
* {@link DefaultCompoundFileDirectory}.
* @lucene.experimental
*/
- public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
- return new DefaultCompoundFileDirectory(this, name, bufferSize, false);
+ public CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException {
+ return new DefaultCompoundFileDirectory(this, name, context, false);
}
/**
@@ -138,8 +132,8 @@ public abstract class Directory implemen
* {@link DefaultCompoundFileDirectory}.
* @lucene.experimental
*/
- public CompoundFileDirectory createCompoundOutput(String name) throws IOException {
- return new DefaultCompoundFileDirectory(this, name, 1024, true);
+ public CompoundFileDirectory createCompoundOutput(String name, IOContext context) throws IOException {
+ return new DefaultCompoundFileDirectory(this, name, context, true);
}
/** Construct a {@link Lock}.
@@ -223,13 +217,13 @@ public abstract class Directory implemen
* <b>NOTE:</b> this method does not check whether <i>dest<i> exist and will
* overwrite it if it does.
*/
- public void copy(Directory to, String src, String dest) throws IOException {
+ public void copy(Directory to, String src, String dest, IOContext context) throws IOException {
IndexOutput os = null;
IndexInput is = null;
IOException priorException = null;
try {
- os = to.createOutput(dest);
- is = openInput(src);
+ os = to.createOutput(dest, context);
+ is = openInput(src, context);
is.copyBytes(os, is.length());
} catch (IOException ioe) {
priorException = ioe;
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FSDirectory.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FSDirectory.java Fri Jul 8 21:02:20 2011
@@ -122,6 +122,10 @@ public abstract class FSDirectory extend
protected final Set<String> staleFiles = synchronizedSet(new HashSet<String>()); // Files written, but not yet sync'ed
private int chunkSize = DEFAULT_READ_CHUNK_SIZE; // LUCENE-1566
+ // null means no limite
+ private Double maxMergeWriteMBPerSec;
+ private RateLimiter mergeWriteRateLimiter;
+
// returns the canonical version of the directory, creating it if it doesn't exist.
private static File getCanonicalPath(File file) throws IOException {
return new File(file.getCanonicalPath());
@@ -286,11 +290,40 @@ public abstract class FSDirectory extend
/** Creates an IndexOutput for the file with the given name. */
@Override
- public IndexOutput createOutput(String name) throws IOException {
+ public IndexOutput createOutput(String name, IOContext context) throws IOException {
ensureOpen();
ensureCanWrite(name);
- return new FSIndexOutput(this, name);
+ return new FSIndexOutput(this, name, context.context == IOContext.Context.MERGE ? mergeWriteRateLimiter : null);
+ }
+
+ /** Sets the maximum (approx) MB/sec allowed by all write
+ * IO performed by merging. Pass null to have no limit.
+ *
+ * <p><b>NOTE</b>: if merges are already running there is
+ * no guarantee this new rate will apply to them; it will
+ * only apply for certain to new merges.
+ *
+ * @lucene.experimental */
+ public synchronized void setMaxMergeWriteMBPerSec(Double mbPerSec) {
+ maxMergeWriteMBPerSec = mbPerSec;
+ if (mbPerSec == null) {
+ if (mergeWriteRateLimiter != null) {
+ mergeWriteRateLimiter.setMaxRate(Double.MAX_VALUE);
+ mergeWriteRateLimiter = null;
+ }
+ } else if (mergeWriteRateLimiter != null) {
+ mergeWriteRateLimiter.setMaxRate(mbPerSec);
+ } else {
+ mergeWriteRateLimiter = new RateLimiter(mbPerSec);
+ }
+ }
+
+ /** See {@link #setMaxMergeWriteMBPerSec}.
+ *
+ * @lucene.experimental */
+ public Double getMaxMergeWriteMBPerSec() {
+ return maxMergeWriteMBPerSec;
}
protected void ensureCanWrite(String name) throws IOException {
@@ -319,13 +352,6 @@ public abstract class FSDirectory extend
staleFiles.removeAll(toSync);
}
- // Inherit javadoc
- @Override
- public IndexInput openInput(String name) throws IOException {
- ensureOpen();
- return openInput(name, BufferedIndexInput.BUFFER_SIZE);
- }
-
@Override
public String getLockID() {
ensureOpen();
@@ -409,17 +435,22 @@ public abstract class FSDirectory extend
private final String name;
private final RandomAccessFile file;
private volatile boolean isOpen; // remember if the file is open, so that we don't try to close it more than once
-
- public FSIndexOutput(FSDirectory parent, String name) throws IOException {
+ private final RateLimiter rateLimiter;
+
+ public FSIndexOutput(FSDirectory parent, String name, RateLimiter rateLimiter) throws IOException {
this.parent = parent;
this.name = name;
file = new RandomAccessFile(new File(parent.directory, name), "rw");
isOpen = true;
+ this.rateLimiter = rateLimiter;
}
/** output methods: */
@Override
public void flushBuffer(byte[] b, int offset, int size) throws IOException {
+ if (rateLimiter != null) {
+ rateLimiter.pause(size);
+ }
file.write(b, offset, size);
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java Fri Jul 8 21:02:20 2011
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Set;
import java.util.HashSet;
+
/**
* Expert: A Directory instance that switches files between
* two other Directory instances.
@@ -125,8 +126,8 @@ public class FileSwitchDirectory extends
}
@Override
- public IndexOutput createOutput(String name) throws IOException {
- return getDirectory(name).createOutput(name);
+ public IndexOutput createOutput(String name, IOContext context) throws IOException {
+ return getDirectory(name).createOutput(name, context);
}
@Override
@@ -145,17 +146,17 @@ public class FileSwitchDirectory extends
}
@Override
- public IndexInput openInput(String name) throws IOException {
- return getDirectory(name).openInput(name);
+ public IndexInput openInput(String name, IOContext context) throws IOException {
+ return getDirectory(name).openInput(name, context);
}
@Override
- public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
- return getDirectory(name).openCompoundInput(name, bufferSize);
+ public CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException {
+ return getDirectory(name).openCompoundInput(name, context);
}
@Override
- public CompoundFileDirectory createCompoundOutput(String name) throws IOException {
- return getDirectory(name).createCompoundOutput(name);
+ public CompoundFileDirectory createCompoundOutput(String name, IOContext context) throws IOException {
+ return getDirectory(name).createCompoundOutput(name, context);
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/MMapDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/MMapDirectory.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/MMapDirectory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/MMapDirectory.java Fri Jul 8 21:02:20 2011
@@ -209,7 +209,7 @@ public class MMapDirectory extends FSDir
/** Creates an IndexInput for the file with the given name. */
@Override
- public IndexInput openInput(String name, int bufferSize) throws IOException {
+ public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
File f = new File(getDirectory(), name);
RandomAccessFile raf = new RandomAccessFile(f, "r");
@@ -221,15 +221,15 @@ public class MMapDirectory extends FSDir
}
@Override
- public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
- return new MMapCompoundFileDirectory(name, bufferSize);
+ public CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException {
+ return new MMapCompoundFileDirectory(name, context);
}
private final class MMapCompoundFileDirectory extends CompoundFileDirectory {
private RandomAccessFile raf = null;
- public MMapCompoundFileDirectory(String fileName, int readBufferSize) throws IOException {
- super(MMapDirectory.this, fileName, readBufferSize);
+ public MMapCompoundFileDirectory(String fileName, IOContext context) throws IOException {
+ super(MMapDirectory.this, fileName, context);
IndexInput stream = null;
try {
File f = new File(MMapDirectory.this.getDirectory(), fileName);
@@ -438,4 +438,5 @@ public class MMapDirectory extends FSDir
}
}
}
+
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java Fri Jul 8 21:02:20 2011
@@ -76,22 +76,22 @@ public class NIOFSDirectory extends FSDi
/** Creates an IndexInput for the file with the given name. */
@Override
- public IndexInput openInput(String name, int bufferSize) throws IOException {
+ public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
- return new NIOFSIndexInput(new File(getDirectory(), name), bufferSize, getReadChunkSize());
+ return new NIOFSIndexInput(new File(getDirectory(), name), context, getReadChunkSize());
}
@Override
- public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
- return new NIOFSCompoundFileDirectory(name, bufferSize);
+ public CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException {
+ return new NIOFSCompoundFileDirectory(name, context);
}
private final class NIOFSCompoundFileDirectory extends CompoundFileDirectory {
private SimpleFSIndexInput.Descriptor fd;
private FileChannel fc;
- public NIOFSCompoundFileDirectory(String fileName, int readBufferSize) throws IOException {
- super(NIOFSDirectory.this, fileName, readBufferSize);
+ public NIOFSCompoundFileDirectory(String fileName, IOContext context) throws IOException {
+ super(NIOFSDirectory.this, fileName, context);
IndexInput stream = null;
try {
File f = new File(NIOFSDirectory.this.getDirectory(), fileName);
@@ -131,8 +131,8 @@ public class NIOFSDirectory extends FSDi
final FileChannel channel;
- public NIOFSIndexInput(File path, int bufferSize, int chunkSize) throws IOException {
- super(path, bufferSize, chunkSize);
+ public NIOFSIndexInput(File path, IOContext context, int chunkSize) throws IOException {
+ super(path, context, chunkSize);
channel = file.getChannel();
}
@@ -229,4 +229,5 @@ public class NIOFSDirectory extends FSDi
}
}
}
+
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/RAMDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/RAMDirectory.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/RAMDirectory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/RAMDirectory.java Fri Jul 8 21:02:20 2011
@@ -27,6 +27,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
+
/**
* A memory-resident {@link Directory} implementation. Locking
* implementation is by default the {@link SingleInstanceLockFactory}
@@ -38,7 +39,7 @@ public class RAMDirectory extends Direct
// *****
// Lock acquisition sequence: RAMDirectory, then RAMFile
- // *****
+ // *****
/** Constructs an empty {@link Directory}. */
public RAMDirectory() {
@@ -65,14 +66,14 @@ public class RAMDirectory extends Direct
* @param dir a <code>Directory</code> value
* @exception IOException if an error occurs
*/
- public RAMDirectory(Directory dir) throws IOException {
- this(dir, false);
+ public RAMDirectory(Directory dir, IOContext context) throws IOException {
+ this(dir, false, context);
}
- private RAMDirectory(Directory dir, boolean closeDir) throws IOException {
+ private RAMDirectory(Directory dir, boolean closeDir, IOContext context) throws IOException {
this();
for (String file : dir.listAll()) {
- dir.copy(this, file, file);
+ dir.copy(this, file, file, context);
}
if (closeDir) {
dir.close();
@@ -149,7 +150,7 @@ public class RAMDirectory extends Direct
/** Creates a new, empty file in the directory with the given name. Returns a stream writing this file. */
@Override
- public IndexOutput createOutput(String name) throws IOException {
+ public IndexOutput createOutput(String name, IOContext context) throws IOException {
ensureOpen();
RAMFile file = newRAMFile();
RAMFile existing = fileMap.remove(name);
@@ -176,7 +177,7 @@ public class RAMDirectory extends Direct
/** Returns a stream reading an existing file. */
@Override
- public IndexInput openInput(String name) throws IOException {
+ public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
RAMFile file = fileMap.get(name);
if (file == null) {
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java Fri Jul 8 21:02:20 2011
@@ -23,6 +23,7 @@ import java.io.RandomAccessFile;
import org.apache.lucene.util.IOUtils;
+
/** A straightforward implementation of {@link FSDirectory}
* using java.io.RandomAccessFile. However, this class has
* poor concurrent performance (multiple threads will
@@ -53,21 +54,21 @@ public class SimpleFSDirectory extends F
/** Creates an IndexInput for the file with the given name. */
@Override
- public IndexInput openInput(String name, int bufferSize) throws IOException {
+ public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
- return new SimpleFSIndexInput(new File(directory, name), bufferSize, getReadChunkSize());
+ return new SimpleFSIndexInput(new File(directory, name), context, getReadChunkSize());
}
@Override
- public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
- return new SimpleFSCompoundFileDirectory(name, bufferSize);
+ public CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException {
+ return new SimpleFSCompoundFileDirectory(name, context);
}
private final class SimpleFSCompoundFileDirectory extends CompoundFileDirectory {
private SimpleFSIndexInput.Descriptor fd;
- public SimpleFSCompoundFileDirectory(String fileName, int readBufferSize) throws IOException {
- super(SimpleFSDirectory.this, fileName, readBufferSize);
+ public SimpleFSCompoundFileDirectory(String fileName, IOContext context) throws IOException {
+ super(SimpleFSDirectory.this, fileName, context);
IndexInput stream = null;
try {
final File f = new File(SimpleFSDirectory.this.getDirectory(), fileName);
@@ -128,8 +129,8 @@ public class SimpleFSDirectory extends F
protected final long off;
protected final long end;
- public SimpleFSIndexInput(File path, int bufferSize, int chunkSize) throws IOException {
- super(bufferSize);
+ public SimpleFSIndexInput(File path, IOContext context, int chunkSize) throws IOException {
+ super(context);
this.file = new Descriptor(path, "r");
this.chunkSize = chunkSize;
this.off = 0L;
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/ArrayUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/ArrayUtil.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/ArrayUtil.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/ArrayUtil.java Fri Jul 8 21:02:20 2011
@@ -230,6 +230,7 @@ public final class ArrayUtil {
}
public static short[] grow(short[] array, int minSize) {
+ assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
short[] newArray = new short[oversize(minSize, RamUsageEstimator.NUM_BYTES_SHORT)];
System.arraycopy(array, 0, newArray, 0, array.length);
@@ -243,6 +244,7 @@ public final class ArrayUtil {
}
public static float[] grow(float[] array, int minSize) {
+ assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
float[] newArray = new float[oversize(minSize, RamUsageEstimator.NUM_BYTES_FLOAT)];
System.arraycopy(array, 0, newArray, 0, array.length);
@@ -256,6 +258,7 @@ public final class ArrayUtil {
}
public static double[] grow(double[] array, int minSize) {
+ assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
double[] newArray = new double[oversize(minSize, RamUsageEstimator.NUM_BYTES_DOUBLE)];
System.arraycopy(array, 0, newArray, 0, array.length);
@@ -269,6 +272,7 @@ public final class ArrayUtil {
}
public static short[] shrink(short[] array, int targetSize) {
+ assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_SHORT);
if (newSize != array.length) {
short[] newArray = new short[newSize];
@@ -279,6 +283,7 @@ public final class ArrayUtil {
}
public static int[] grow(int[] array, int minSize) {
+ assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
int[] newArray = new int[oversize(minSize, RamUsageEstimator.NUM_BYTES_INT)];
System.arraycopy(array, 0, newArray, 0, array.length);
@@ -292,6 +297,7 @@ public final class ArrayUtil {
}
public static int[] shrink(int[] array, int targetSize) {
+ assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_INT);
if (newSize != array.length) {
int[] newArray = new int[newSize];
@@ -302,6 +308,7 @@ public final class ArrayUtil {
}
public static long[] grow(long[] array, int minSize) {
+ assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
long[] newArray = new long[oversize(minSize, RamUsageEstimator.NUM_BYTES_LONG)];
System.arraycopy(array, 0, newArray, 0, array.length);
@@ -315,6 +322,7 @@ public final class ArrayUtil {
}
public static long[] shrink(long[] array, int targetSize) {
+ assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_LONG);
if (newSize != array.length) {
long[] newArray = new long[newSize];
@@ -325,6 +333,7 @@ public final class ArrayUtil {
}
public static byte[] grow(byte[] array, int minSize) {
+ assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
byte[] newArray = new byte[oversize(minSize, 1)];
System.arraycopy(array, 0, newArray, 0, array.length);
@@ -338,6 +347,7 @@ public final class ArrayUtil {
}
public static byte[] shrink(byte[] array, int targetSize) {
+ assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, 1);
if (newSize != array.length) {
byte[] newArray = new byte[newSize];
@@ -348,6 +358,7 @@ public final class ArrayUtil {
}
public static boolean[] grow(boolean[] array, int minSize) {
+ assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
boolean[] newArray = new boolean[oversize(minSize, 1)];
System.arraycopy(array, 0, newArray, 0, array.length);
@@ -361,6 +372,7 @@ public final class ArrayUtil {
}
public static boolean[] shrink(boolean[] array, int targetSize) {
+ assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, 1);
if (newSize != array.length) {
boolean[] newArray = new boolean[newSize];
@@ -371,6 +383,7 @@ public final class ArrayUtil {
}
public static char[] grow(char[] array, int minSize) {
+ assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
char[] newArray = new char[oversize(minSize, RamUsageEstimator.NUM_BYTES_CHAR)];
System.arraycopy(array, 0, newArray, 0, array.length);
@@ -384,6 +397,7 @@ public final class ArrayUtil {
}
public static char[] shrink(char[] array, int targetSize) {
+ assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_CHAR);
if (newSize != array.length) {
char[] newArray = new char[newSize];
@@ -394,6 +408,7 @@ public final class ArrayUtil {
}
public static int[][] grow(int[][] array, int minSize) {
+ assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
int[][] newArray = new int[oversize(minSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF)][];
System.arraycopy(array, 0, newArray, 0, array.length);
@@ -408,6 +423,7 @@ public final class ArrayUtil {
}
public static int[][] shrink(int[][] array, int targetSize) {
+ assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
if (newSize != array.length) {
int[][] newArray = new int[newSize][];
@@ -419,6 +435,7 @@ public final class ArrayUtil {
}
public static float[][] grow(float[][] array, int minSize) {
+ assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
float[][] newArray = new float[oversize(minSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF)][];
System.arraycopy(array, 0, newArray, 0, array.length);
@@ -433,6 +450,7 @@ public final class ArrayUtil {
}
public static float[][] shrink(float[][] array, int targetSize) {
+ assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
if (newSize != array.length) {
float[][] newArray = new float[newSize][];
@@ -494,6 +512,7 @@ public final class ArrayUtil {
/* DISABLE THIS FOR NOW: This has performance problems until Java creates intrinsics for Class#getComponentType() and Array.newInstance()
public static <T> T[] grow(T[] array, int minSize) {
+ assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
@SuppressWarnings("unchecked") final T[] newArray =
(T[]) Array.newInstance(array.getClass().getComponentType(), oversize(minSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
@@ -508,6 +527,7 @@ public final class ArrayUtil {
}
public static <T> T[] shrink(T[] array, int targetSize) {
+ assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
if (newSize != array.length) {
@SuppressWarnings("unchecked") final T[] newArray =
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/BitVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/BitVector.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/BitVector.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/BitVector.java Fri Jul 8 21:02:20 2011
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Arrays;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@@ -214,9 +215,9 @@ public final class BitVector implements
/** Writes this vector to the file <code>name</code> in Directory
<code>d</code>, in a format that can be read by the constructor {@link
- #BitVector(Directory, String)}. */
- public final void write(Directory d, String name) throws IOException {
- IndexOutput output = d.createOutput(name);
+ #BitVector(Directory, String, IOContext)}. */
+ public final void write(Directory d, String name, IOContext context) throws IOException {
+ IndexOutput output = d.createOutput(name, context);
try {
output.writeInt(-2);
CodecUtil.writeHeader(output, CODEC, VERSION_CURRENT);
@@ -328,8 +329,8 @@ public final class BitVector implements
/** Constructs a bit vector from the file <code>name</code> in Directory
<code>d</code>, as written by the {@link #write} method.
*/
- public BitVector(Directory d, String name) throws IOException {
- IndexInput input = d.openInput(name);
+ public BitVector(Directory d, String name, IOContext context) throws IOException {
+ IndexInput input = d.openInput(name, context);
try {
final int firstInt = input.readInt();
Modified: lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java Fri Jul 8 21:02:20 2011
@@ -84,8 +84,8 @@ public class MockFixedIntBlockCodec exte
}
@Override
- public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException {
- return new FixedIntBlockIndexInput(dir.openInput(fileName, readBufferSize)) {
+ public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException {
+ return new FixedIntBlockIndexInput(dir.openInput(fileName, context)) {
@Override
protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) throws IOException {
@@ -102,8 +102,8 @@ public class MockFixedIntBlockCodec exte
}
@Override
- public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException {
- IndexOutput out = dir.createOutput(fileName);
+ public IntIndexOutput createOutput(Directory dir, String fileName, IOContext context) throws IOException {
+ IndexOutput out = dir.createOutput(fileName, context);
boolean success = false;
try {
FixedIntBlockIndexOutput ret = new FixedIntBlockIndexOutput(out, blockSize) {
@@ -160,7 +160,7 @@ public class MockFixedIntBlockCodec exte
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir,
state.segmentInfo,
- state.readBufferSize,
+ state.context,
new MockIntFactory(blockSize), state.codecId);
TermsIndexReaderBase indexReader;
@@ -170,7 +170,8 @@ public class MockFixedIntBlockCodec exte
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
- BytesRef.getUTF8SortedAsUnicodeComparator(), state.codecId);
+ BytesRef.getUTF8SortedAsUnicodeComparator(), state.codecId,
+ IOContext.DEFAULT);
success = true;
} finally {
if (!success) {
@@ -185,7 +186,7 @@ public class MockFixedIntBlockCodec exte
state.fieldInfos,
state.segmentInfo.name,
postingsReader,
- state.readBufferSize,
+ state.context,
StandardCodec.TERMS_CACHE_SIZE,
state.codecId);
success = true;
@@ -224,6 +225,6 @@ public class MockFixedIntBlockCodec exte
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator());
+ return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java?rev=1144510&r1=1144509&r2=1144510&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java Fri Jul 8 21:02:20 2011
@@ -48,6 +48,7 @@ import org.apache.lucene.index.codecs.Bl
import org.apache.lucene.index.codecs.TermsIndexReaderBase;
import org.apache.lucene.index.codecs.TermsIndexWriterBase;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
@@ -82,8 +83,8 @@ public class MockVariableIntBlockCodec e
}
@Override
- public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException {
- final IndexInput in = dir.openInput(fileName, readBufferSize);
+ public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException {
+ final IndexInput in = dir.openInput(fileName, context);
final int baseBlockSize = in.readInt();
return new VariableIntBlockIndexInput(in) {
@@ -106,8 +107,8 @@ public class MockVariableIntBlockCodec e
}
@Override
- public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException {
- final IndexOutput out = dir.createOutput(fileName);
+ public IntIndexOutput createOutput(Directory dir, String fileName, IOContext context) throws IOException {
+ final IndexOutput out = dir.createOutput(fileName, context);
boolean success = false;
try {
out.writeInt(baseBlockSize);
@@ -182,7 +183,7 @@ public class MockVariableIntBlockCodec e
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir,
state.segmentInfo,
- state.readBufferSize,
+ state.context,
new MockIntFactory(baseBlockSize), state.codecId);
TermsIndexReaderBase indexReader;
@@ -193,7 +194,7 @@ public class MockVariableIntBlockCodec e
state.segmentInfo.name,
state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUnicodeComparator(),
- state.codecId);
+ state.codecId, state.context);
success = true;
} finally {
if (!success) {
@@ -208,7 +209,7 @@ public class MockVariableIntBlockCodec e
state.fieldInfos,
state.segmentInfo.name,
postingsReader,
- state.readBufferSize,
+ state.context,
StandardCodec.TERMS_CACHE_SIZE,
state.codecId);
success = true;
@@ -247,6 +248,6 @@ public class MockVariableIntBlockCodec e
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator());
+ return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
}
}