You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2010/07/30 08:13:14 UTC
svn commit: r980656 - in /lucene/dev/trunk: ./ lucene/ lucene/contrib/
lucene/contrib/highlighter/src/test/
lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/
lucene/src/java/org/apache/lucene/analysis/ lucene/src/java/org/apach...
Author: shaie
Date: Fri Jul 30 06:13:13 2010
New Revision: 980656
URL: http://svn.apache.org/viewvc?rev=980656&view=rev
Log:
LUCENE-2574: Optimize copies between IndexInput and Output
Modified:
lucene/dev/trunk/ (props changed)
lucene/dev/trunk/lucene/ (props changed)
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/build.xml (props changed)
lucene/dev/trunk/lucene/contrib/ (props changed)
lucene/dev/trunk/lucene/contrib/CHANGES.txt (props changed)
lucene/dev/trunk/lucene/contrib/highlighter/src/test/ (props changed)
lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (props changed)
lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java (props changed)
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CompoundFileReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (props changed)
lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/Directory.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/FSDirectory.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/IndexInput.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/RAMInputStream.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/RAMOutputStream.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java (props changed)
lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java (props changed)
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java (props changed)
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java (props changed)
lucene/dev/trunk/solr/ (props changed)
lucene/dev/trunk/solr/lib/commons-httpclient-3.1.jar (props changed)
lucene/dev/trunk/solr/lib/jcl-over-slf4j-1.5.5.jar (props changed)
lucene/dev/trunk/solr/src/common/org/apache/solr/common/ (props changed)
lucene/dev/trunk/solr/src/maven/solr-core-pom.xml.template (props changed)
lucene/dev/trunk/solr/src/maven/solr-solrj-pom.xml.template (props changed)
lucene/dev/trunk/solr/src/solrj/org/ (props changed)
lucene/dev/trunk/solr/src/test/org/apache/solr/client/ (props changed)
lucene/dev/trunk/solr/src/webapp/src/org/apache/solr/client/solrj/embedded/ (props changed)
Propchange: lucene/dev/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,2 +1,2 @@
-/lucene/dev/branches/branch_3x:949730,957490,961612,979161
+/lucene/dev/branches/branch_3x:949730,957490,961612,979161,980654
/lucene/dev/branches/preflexfixes:967125-979432
Propchange: lucene/dev/trunk/lucene/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,4 +1,4 @@
-/lucene/dev/branches/branch_3x/lucene:943137,949730,957490,960490,961612,979161
+/lucene/dev/branches/branch_3x/lucene:943137,949730,957490,960490,961612,979161,980654
/lucene/dev/branches/preflexfixes/lucene:967125-979432
/lucene/java/branches/flex_1458:824912-931101
/lucene/java/branches/lucene_2_4:748824
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=980656&r1=980655&r2=980656&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Fri Jul 30 06:13:13 2010
@@ -204,7 +204,11 @@ Optimizations
* LUCENE-2531: Fix issue when sorting by a String field that was
causing too many fallbacks to compare-by-value (instead of by-ord).
(Mike McCandless)
-
+
+* LUCENE-2574: IndexInput exposes copyBytes(IndexOutput, long) to allow for
+ efficient copying by sub-classes. Optimized copy is implemented for RAM and FS
+ streams. (Shai Erera)
+
======================= Lucene 3.x (not yet released) =======================
Changes in backwards compatibility policy
Propchange: lucene/dev/trunk/lucene/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,4 +1,4 @@
-/lucene/dev/branches/branch_3x/lucene/build.xml:943137,949730,957490,960490,961612,979161
+/lucene/dev/branches/branch_3x/lucene/build.xml:943137,949730,957490,960490,961612,979161,980654
/lucene/dev/branches/preflexfixes/lucene/build.xml:967125-979432
/lucene/java/branches/flex_1458/build.xml:824912-931101
/lucene/java/branches/lucene_2_9/build.xml:909334,948516
Propchange: lucene/dev/trunk/lucene/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,4 +1,4 @@
-/lucene/dev/branches/branch_3x/lucene/contrib:943137,949730,957490,960490,961612,979161
+/lucene/dev/branches/branch_3x/lucene/contrib:943137,949730,957490,960490,961612,979161,980654
/lucene/dev/branches/preflexfixes/lucene/contrib:967125-979432
/lucene/java/branches/flex_1458/contrib:824912-931101
/lucene/java/branches/lucene_2_4/contrib:748824
Propchange: lucene/dev/trunk/lucene/contrib/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,4 +1,4 @@
-/lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt:943137,949730,957490,960490,961612,979161
+/lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt:943137,949730,957490,960490,961612,979161,980654
/lucene/dev/branches/preflexfixes/lucene/contrib/CHANGES.txt:967125-979432
/lucene/java/branches/flex_1458/contrib/CHANGES.txt:824912-931101
/lucene/java/branches/lucene_2_4/contrib/CHANGES.txt:748824
Propchange: lucene/dev/trunk/lucene/contrib/highlighter/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,4 +1,4 @@
-/lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test:943137,949730,957490,960490,961612,979161
+/lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test:943137,949730,957490,960490,961612,979161,980654
/lucene/dev/branches/preflexfixes/lucene/contrib/highlighter/src/test:967125-979432
/lucene/java/branches/flex_1458/contrib/highlighter/src/test:824912-931101
/lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824
Propchange: lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,4 +1,4 @@
-/lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:943137,949730,957490,960490,961612,979161
+/lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:943137,949730,957490,960490,961612,979161,980654
/lucene/dev/branches/preflexfixes/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:967125-979432
/lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:824912-931101
/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:909334,948516
Propchange: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,4 +1,4 @@
-/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:943137,949730,957490,960490,961612,979161
+/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:943137,949730,957490,960490,961612,979161,980654
/lucene/dev/branches/preflexfixes/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:967125-979432
/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java:824912-931101
/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:909334,948516
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CompoundFileReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CompoundFileReader.java?rev=980656&r1=980655&r2=980656&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CompoundFileReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CompoundFileReader.java Fri Jul 30 06:13:13 2010
@@ -303,6 +303,17 @@ public class CompoundFileReader extends
return length;
}
+ @Override
+ public void copyBytes(IndexOutput out, long numBytes) throws IOException {
+ // Copy first whatever is in the buffer
+ numBytes -= flushBuffer(out, numBytes);
+
+ // If there are more bytes left to copy, delegate the copy task to the
+ // base IndexInput, in case it can do an optimized copy.
+ if (numBytes > 0) {
+ base.copyBytes(out, numBytes);
+ }
+ }
}
Propchange: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,4 +1,4 @@
-/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:943137,949730,957490,960490,961612,979161
+/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:943137,949730,957490,960490,961612,979161,980654
/lucene/dev/branches/preflexfixes/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:967125-979432
/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:824912-931101
/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:909334,948516
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java?rev=980656&r1=980655&r2=980656&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java Fri Jul 30 06:13:13 2010
@@ -26,9 +26,9 @@ public abstract class BufferedIndexInput
public static final int BUFFER_SIZE = 1024;
private int bufferSize = BUFFER_SIZE;
-
+
protected byte[] buffer;
-
+
private long bufferStart = 0; // position in file of buffer
private int bufferLength = 0; // end of valid bytes
private int bufferPosition = 0; // next byte to read
@@ -205,4 +205,37 @@ public abstract class BufferedIndexInput
return clone;
}
+ /**
+ * Flushes the in-memory bufer to the given output, copying at most
+ * <code>numBytes</code>.
+ * <p>
+ * <b>NOTE:</b> this method does not refill the buffer, however it does
+ * advance the buffer position.
+ *
+ * @return the number of bytes actually flushed from the in-memory buffer.
+ */
+ protected int flushBuffer(IndexOutput out, long numBytes) throws IOException {
+ int toCopy = bufferLength - bufferPosition;
+ if (toCopy > numBytes) {
+ toCopy = (int) numBytes;
+ }
+ if (toCopy > 0) {
+ out.writeBytes(buffer, bufferPosition, toCopy);
+ bufferPosition += toCopy;
+ }
+ return toCopy;
+ }
+
+ @Override
+ public void copyBytes(IndexOutput out, long numBytes) throws IOException {
+ assert numBytes >= 0: "numBytes=" + numBytes;
+
+ while (numBytes > 0) {
+ if (bufferLength == bufferPosition) {
+ refill();
+ }
+ numBytes -= flushBuffer(out, numBytes);
+ }
+ }
+
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/Directory.java?rev=980656&r1=980655&r2=980656&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/Directory.java Fri Jul 30 06:13:13 2010
@@ -215,26 +215,11 @@ public abstract class Directory implemen
* overwrite it if it does.
*/
public void copy(Directory to, String src, String dest) throws IOException {
- IndexOutput os = null;
- IndexInput is = null;
+ IndexOutput os = to.createOutput(dest);
+ IndexInput is = openInput(src);
IOException priorException = null;
- int bufSize = BufferedIndexOutput.BUFFER_SIZE;
- byte[] buf = new byte[bufSize];
try {
- // create file in dest directory
- os = to.createOutput(dest);
- // read current file
- is = openInput(src);
- // and copy to dest directory
- long len = is.length();
- long numRead = 0;
- while (numRead < len) {
- long left = len - numRead;
- int toRead = (int) (bufSize < left ? bufSize : left);
- is.readBytes(buf, 0, toRead);
- os.writeBytes(buf, toRead);
- numRead += toRead;
- }
+ is.copyBytes(os, is.length());
} catch (IOException ioe) {
priorException = ioe;
} finally {
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/FSDirectory.java?rev=980656&r1=980655&r2=980656&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/FSDirectory.java Fri Jul 30 06:13:13 2010
@@ -18,9 +18,7 @@ package org.apache.lucene.store;
*/
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.RandomAccessFile;
@@ -35,7 +33,6 @@ import java.util.HashSet;
import java.util.Set;
import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.Constants;
@@ -438,47 +435,6 @@ public abstract class FSDirectory extend
return chunkSize;
}
- @Override
- public void copy(Directory to, String src, String dest) throws IOException {
- if (to instanceof FSDirectory) {
- FSDirectory target = (FSDirectory) to;
- target.ensureCanWrite(dest);
- FileChannel input = null;
- FileChannel output = null;
- IOException priorException = null;
- try {
- input = new FileInputStream(new File(directory, src)).getChannel();
- output = new FileOutputStream(new File(target.directory, dest)).getChannel();
- copy(input, output, input.size());
- } catch (IOException ioe) {
- priorException = ioe;
- } finally {
- IOUtils.closeSafely(priorException, input, output);
- }
- } else {
- super.copy(to, src, dest);
- }
- }
-
- /**
- * Copies the content of a given {@link FileChannel} to a destination one. The
- * copy is done in chunks of 2MB because if transferFrom is used without a
- * limit when copying a very large file, then an OOM may be thrown (depends on
- * the state of the RAM in the machine, as well as the OS used). Performance
- * measurements showed that chunk sizes larger than 2MB do not result in much
- * faster file copy, therefore we limit the size to be safe with different
- * file sizes and systems.
- */
- static void copy(FileChannel input, FileChannel output, long numBytes) throws IOException {
- long pos = output.position();
- long writeTo = numBytes + pos;
- while (pos < writeTo) {
- pos += output.transferFrom(input, pos, Math.min(CHANNEL_CHUNK_SIZE, writeTo - pos));
- }
- // transferFrom does not change the position of the channel. Need to change it manually
- output.position(pos);
- }
-
protected static class FSIndexOutput extends BufferedIndexOutput {
private final FSDirectory parent;
private final String name;
@@ -501,23 +457,37 @@ public abstract class FSDirectory extend
@Override
public void copyBytes(DataInput input, long numBytes) throws IOException {
// Optimized copy only if the number of bytes to copy is larger than the
- // buffer size, and the given IndexInput supports FileChannel copying ..
+ // buffer size, and the given IndexInput supports FileChannel copying.
// NOTE: the below check relies on NIOIndexInput extending Simple. If that
// changes in the future, we should change the check as well.
- if (numBytes > BUFFER_SIZE && input instanceof SimpleFSIndexInput) {
- // flush any bytes in the buffer
- flush();
- // do the optimized copy
- FileChannel in = ((SimpleFSIndexInput) input).file.getChannel();
- FileChannel out = file.getChannel();
- copy(in, out, numBytes);
- // corrects the position in super (BufferedIndexOutput), so that calls
- // to getFilePointer will return the correct pointer.
- // Perhaps a specific method is better?
- super.seek(out.position());
- } else {
+ if (numBytes <= BUFFER_SIZE || !(input instanceof SimpleFSIndexInput)) {
super.copyBytes(input, numBytes);
+ return;
+ }
+
+ SimpleFSIndexInput fsInput = (SimpleFSIndexInput) input;
+
+ // flush any bytes in the buffer
+ flush();
+
+ // flush any bytes in the input's buffer.
+ numBytes -= fsInput.flushBuffer(this, numBytes);
+
+ // do the optimized copy
+ FileChannel in = fsInput.file.getChannel();
+ FileChannel out = file.getChannel();
+ long pos = out.position();
+ long writeTo = numBytes + pos;
+ while (pos < writeTo) {
+ pos += out.transferFrom(in, pos, Math.min(CHANNEL_CHUNK_SIZE, writeTo - pos));
}
+ // transferFrom does not change the position of the channel. Need to change it manually
+ out.position(pos);
+
+ // corrects the position in super (BufferedIndexOutput), so that calls
+ // to getFilePointer will return the correct pointer.
+ // Perhaps a specific method is better?
+ super.seek(out.position());
}
@Override
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/IndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/IndexInput.java?rev=980656&r1=980655&r2=980656&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/IndexInput.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/IndexInput.java Fri Jul 30 06:13:13 2010
@@ -25,6 +25,9 @@ import java.io.IOException;
* @see Directory
*/
public abstract class IndexInput extends DataInput implements Cloneable,Closeable {
+
+ protected byte[] copyBuf = null;
+
/** Closes the stream to further operations. */
public abstract void close() throws IOException;
@@ -41,4 +44,31 @@ public abstract class IndexInput extends
/** The number of bytes in the file. */
public abstract long length();
-}
\ No newline at end of file
+
+ /**
+ * Copies <code>numBytes</code> bytes to the given {@link IndexOutput}.
+ * <p>
+ * <b>NOTE:</b> this method uses an intermediate buffer to copy the bytes.
+ * Consider overriding it in your implementation, if you can make a better,
+ * optimized copy.
+ * <p>
+ * <b>NOTE</b> ensure that there are enough bytes in the input to copy to
+ * output. Otherwise, different exceptions may be thrown, depending on the
+ * implementation.
+ */
+ public void copyBytes(IndexOutput out, long numBytes) throws IOException {
+ assert numBytes >= 0: "numBytes=" + numBytes;
+
+ if (copyBuf == null) {
+ copyBuf = new byte[BufferedIndexInput.BUFFER_SIZE];
+ }
+
+ while (numBytes > 0) {
+ final int toCopy = (int) (numBytes > copyBuf.length ? copyBuf.length : numBytes);
+ readBytes(copyBuf, 0, toCopy);
+ out.writeBytes(copyBuf, 0, toCopy);
+ numBytes -= toCopy;
+ }
+ }
+
+}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/RAMInputStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/RAMInputStream.java?rev=980656&r1=980655&r2=980656&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/RAMInputStream.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/RAMInputStream.java Fri Jul 30 06:13:13 2010
@@ -1,7 +1,5 @@
package org.apache.lucene.store;
-import java.io.IOException;
-
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -19,10 +17,9 @@ import java.io.IOException;
* limitations under the License.
*/
-/**
- * A memory-resident {@link IndexInput} implementation.
- */
+import java.io.IOException;
+/** A memory-resident {@link IndexInput} implementation. */
class RAMInputStream extends IndexInput implements Cloneable {
static final int BUFFER_SIZE = RAMOutputStream.BUFFER_SIZE;
@@ -105,6 +102,27 @@ class RAMInputStream extends IndexInput
}
@Override
+ public void copyBytes(IndexOutput out, long numBytes) throws IOException {
+ assert numBytes >= 0: "numBytes=" + numBytes;
+
+ long left = numBytes;
+ while (left > 0) {
+ if (bufferPosition == bufferLength) {
+ ++currentBufferIndex;
+ switchCurrentBuffer(true);
+ }
+
+ final int bytesInBuffer = bufferLength - bufferPosition;
+ final int toCopy = (int) (bytesInBuffer < left ? bytesInBuffer : left);
+ out.writeBytes(currentBuffer, bufferPosition, toCopy);
+ bufferPosition += toCopy;
+ left -= toCopy;
+ }
+
+ assert left == 0: "Insufficient bytes to copy: numBytes=" + numBytes + " copied=" + (numBytes - left);
+ }
+
+ @Override
public long getFilePointer() {
return currentBufferIndex < 0 ? 0 : bufferStart + bufferPosition;
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/RAMOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/RAMOutputStream.java?rev=980656&r1=980655&r2=980656&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/RAMOutputStream.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/RAMOutputStream.java Fri Jul 30 06:13:13 2010
@@ -24,7 +24,6 @@ import java.io.IOException;
*
* @lucene.internal
*/
-
public class RAMOutputStream extends IndexOutput {
static final int BUFFER_SIZE = 1024;
@@ -161,4 +160,26 @@ public class RAMOutputStream extends Ind
public long sizeInBytes() {
return file.numBuffers() * BUFFER_SIZE;
}
+
+ @Override
+ public void copyBytes(DataInput input, long numBytes) throws IOException {
+ assert numBytes >= 0: "numBytes=" + numBytes;
+
+ while (numBytes > 0) {
+ if (bufferPosition == bufferLength) {
+ currentBufferIndex++;
+ switchCurrentBuffer();
+ }
+
+ int toCopy = currentBuffer.length - bufferPosition;
+ if (numBytes < toCopy) {
+ toCopy = (int) numBytes;
+ }
+ input.readBytes(currentBuffer, bufferPosition, toCopy, false);
+ numBytes -= toCopy;
+ bufferPosition += toCopy;
+ }
+
+ }
+
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java?rev=980656&r1=980655&r2=980656&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java Fri Jul 30 06:13:13 2010
@@ -160,5 +160,12 @@ public class SimpleFSDirectory extends F
boolean isFDValid() throws IOException {
return file.getFD().valid();
}
+
+ @Override
+ public void copyBytes(IndexOutput out, long numBytes) throws IOException {
+ numBytes -= flushBuffer(out, numBytes);
+ // If out is FSIndexOutput, the copy will be optimized
+ out.copyBytes(this, numBytes);
+ }
}
}
Propchange: lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,4 +1,4 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java:943137,949730,957490,960490,961612,979161
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java:943137,949730,957490,960490,961612,979161,980654
/lucene/dev/branches/preflexfixes/lucene/src/test/org/apache/lucene/document/TestDateTools.java:967125-979432
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestDateTools.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
Propchange: lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,4 +1,4 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:943137,949730,957490,960490,961612,979161
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:943137,949730,957490,960490,961612,979161,980654
/lucene/dev/branches/preflexfixes/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:967125-979432
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestNumberTools.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
Propchange: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,4 +1,4 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:943137,949730,957490,960490,961612,979161
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:943137,949730,957490,960490,961612,979161,980654
/lucene/dev/branches/preflexfixes/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:967125-979432
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
Propchange: lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,4 +1,4 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:943137,949730,957490,960490,961612,979161
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:943137,949730,957490,960490,961612,979161,980654
/lucene/dev/branches/preflexfixes/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:967125-979432
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestAttributeSource.java:824912-931101
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:909334,948516
Propchange: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,3 +1,4 @@
+/lucene/dev/branches/branch_3x/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java:980654
/lucene/dev/branches/preflexfixes/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java:967125-979432
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:824912-931101
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824
Propchange: lucene/dev/trunk/solr/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,4 +1,4 @@
-/lucene/dev/branches/branch_3x/solr:949730,957490*,961612,979161
+/lucene/dev/branches/branch_3x/solr:949730,957490*,961612,979161,980654
/lucene/dev/branches/preflexfixes/solr:967125-979432
/lucene/java/branches/lucene_2_4/solr:748824
/lucene/java/branches/lucene_2_9/solr:817269-818600,825998,829134,829881,831036,896850,909334
Propchange: lucene/dev/trunk/solr/lib/commons-httpclient-3.1.jar
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,3 +1,3 @@
-/lucene/dev/branches/branch_3x/solr/lib/commons-httpclient-3.1.jar:949730,961612,979161
+/lucene/dev/branches/branch_3x/solr/lib/commons-httpclient-3.1.jar:949730,961612,979161,980654
/lucene/dev/branches/preflexfixes/solr/lib/commons-httpclient-3.1.jar:967125-979432
/lucene/solr/trunk/lib/commons-httpclient-3.1.jar:922950-923910,923912-925091
Propchange: lucene/dev/trunk/solr/lib/jcl-over-slf4j-1.5.5.jar
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,3 +1,3 @@
-/lucene/dev/branches/branch_3x/solr/lib/jcl-over-slf4j-1.5.5.jar:949730,961612,979161
+/lucene/dev/branches/branch_3x/solr/lib/jcl-over-slf4j-1.5.5.jar:949730,961612,979161,980654
/lucene/dev/branches/preflexfixes/solr/lib/jcl-over-slf4j-1.5.5.jar:967125-979432
/lucene/solr/trunk/lib/jcl-over-slf4j-1.5.5.jar:922950-923910,923912-925091
Propchange: lucene/dev/trunk/solr/src/common/org/apache/solr/common/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,3 +1,3 @@
-/lucene/dev/branches/branch_3x/solr/src/common/org/apache/solr/common:949730,961612,979161
+/lucene/dev/branches/branch_3x/solr/src/common/org/apache/solr/common:949730,961612,979161,980654
/lucene/dev/branches/preflexfixes/solr/src/common/org/apache/solr/common:967125-979432
/lucene/solr/trunk/src/common/org/apache/solr/common:922950-923910,923912-925091
Propchange: lucene/dev/trunk/solr/src/maven/solr-core-pom.xml.template
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,3 +1,3 @@
-/lucene/dev/branches/branch_3x/solr/src/maven/solr-core-pom.xml.template:949730,961612,979161
+/lucene/dev/branches/branch_3x/solr/src/maven/solr-core-pom.xml.template:949730,961612,979161,980654
/lucene/dev/branches/preflexfixes/solr/src/maven/solr-core-pom.xml.template:967125-979432
/lucene/solr/trunk/src/maven/solr-core-pom.xml.template:922950-923910,923912-925091
Propchange: lucene/dev/trunk/solr/src/maven/solr-solrj-pom.xml.template
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,3 +1,3 @@
-/lucene/dev/branches/branch_3x/solr/src/maven/solr-solrj-pom.xml.template:949730,961612,979161
+/lucene/dev/branches/branch_3x/solr/src/maven/solr-solrj-pom.xml.template:949730,961612,979161,980654
/lucene/dev/branches/preflexfixes/solr/src/maven/solr-solrj-pom.xml.template:967125-979432
/lucene/solr/trunk/src/maven/solr-solrj-pom.xml.template:922950-923910,923912-925091
Propchange: lucene/dev/trunk/solr/src/solrj/org/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,3 +1,3 @@
-/lucene/dev/branches/branch_3x/solr/src/solrj/org:949730,961612,979161
+/lucene/dev/branches/branch_3x/solr/src/solrj/org:949730,961612,979161,980654
/lucene/dev/branches/preflexfixes/solr/src/solrj/org:967125-979432
/lucene/solr/trunk/src/solrj/org:922950-923910,923912-925091
Propchange: lucene/dev/trunk/solr/src/test/org/apache/solr/client/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,3 +1,3 @@
-/lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/client:949730,961612,979161
+/lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/client:949730,961612,979161,980654
/lucene/dev/branches/preflexfixes/solr/src/test/org/apache/solr/client:967125-979432
/lucene/solr/trunk/src/test/org/apache/solr/client:922950-923910,923912-925091
Propchange: lucene/dev/trunk/solr/src/webapp/src/org/apache/solr/client/solrj/embedded/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 30 06:13:13 2010
@@ -1,3 +1,3 @@
-/lucene/dev/branches/branch_3x/solr/src/webapp/src/org/apache/solr/client/solrj/embedded:949730,961612,979161
+/lucene/dev/branches/branch_3x/solr/src/webapp/src/org/apache/solr/client/solrj/embedded:949730,961612,979161,980654
/lucene/dev/branches/preflexfixes/solr/src/webapp/src/org/apache/solr/client/solrj/embedded:967125-979432
/lucene/solr/trunk/src/webapp/src/org/apache/solr/client/solrj/embedded:922950-923910,923912-925091