You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2012/08/18 02:17:11 UTC
svn commit: r1374481 - in /lucene/dev/branches/branch_4x: ./ dev-tools/
lucene/ lucene/analysis/
lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/
lucene/benchmark/ lucene/core/ lucene/demo/ lucene/facet/ lucene/grouping/
luce...
Author: yonik
Date: Sat Aug 18 00:17:09 2012
New Revision: 1374481
URL: http://svn.apache.org/viewvc?rev=1374481&view=rev
Log:
SOLR-3715: fix FastOutputStream, FastWriter when buffer sizes can change
Added:
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/MemOutputStream.java
- copied unchanged from r1374480, lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/MemOutputStream.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/util/TestFastOutputStream.java
- copied unchanged from r1374480, lucene/dev/trunk/solr/core/src/test/org/apache/solr/util/TestFastOutputStream.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/util/TestFastWriter.java
- copied unchanged from r1374480, lucene/dev/trunk/solr/core/src/test/org/apache/solr/util/TestFastWriter.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/BUILD.txt (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/branch_4x/lucene/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/README.txt (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java (props changed)
lucene/dev/branches/branch_4x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/common-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/demo/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/branch_4x/lucene/join/ (props changed)
lucene/dev/branches/branch_4x/lucene/licenses/ (props changed)
lucene/dev/branches/branch_4x/lucene/memory/ (props changed)
lucene/dev/branches/branch_4x/lucene/misc/ (props changed)
lucene/dev/branches/branch_4x/lucene/module-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/queries/ (props changed)
lucene/dev/branches/branch_4x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/lucene/site/ (props changed)
lucene/dev/branches/branch_4x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/tools/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/solr/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/solr/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/README.txt (props changed)
lucene/dev/branches/branch_4x/solr/build.xml (props changed)
lucene/dev/branches/branch_4x/solr/cloud-dev/ (props changed)
lucene/dev/branches/branch_4x/solr/common-build.xml (props changed)
lucene/dev/branches/branch_4x/solr/contrib/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/TransactionLog.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/FastWriter.java
lucene/dev/branches/branch_4x/solr/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/solr/example/ (props changed)
lucene/dev/branches/branch_4x/solr/lib/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/scripts/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/util/FastOutputStream.java
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/testlogging.properties (props changed)
lucene/dev/branches/branch_4x/solr/webapp/ (props changed)
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/TransactionLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/TransactionLog.java?rev=1374481&r1=1374480&r2=1374481&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/TransactionLog.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/TransactionLog.java Sat Aug 18 00:17:09 2012
@@ -775,31 +775,3 @@ class ChannelFastInputStream extends Fas
}
-class MemOutputStream extends FastOutputStream {
- public List<byte[]> buffers = new LinkedList<byte[]>();
- public MemOutputStream(byte[] tempBuffer) {
- super(null, tempBuffer, 0);
- }
-
- @Override
- public void flush(byte[] arr, int offset, int len) throws IOException {
- if (arr == buf && offset==0 && len==buf.length) {
- buffers.add(buf); // steal the buffer
- buf = new byte[8192];
- } else if (len > 0) {
- byte[] newBuf = new byte[len];
- System.arraycopy(arr, offset, newBuf, 0, len);
- buffers.add(newBuf);
- }
- }
-
- public void writeAll(FastOutputStream fos) throws IOException {
- for (byte[] buffer : buffers) {
- fos.write(buffer);
- }
- if (pos > 0) {
- fos.write(buf, 0, pos);
- }
- }
-}
-
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/FastWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/FastWriter.java?rev=1374481&r1=1374480&r2=1374481&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/FastWriter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/FastWriter.java Sat Aug 18 00:17:09 2012
@@ -28,7 +28,7 @@ public class FastWriter extends Writer {
// it won't cause double buffering.
private static final int BUFSIZE = 8192;
protected final Writer sink;
- protected final char[] buf;
+ protected char[] buf;
protected int pos;
public FastWriter(Writer w) {
@@ -69,42 +69,64 @@ public class FastWriter extends Writer {
}
@Override
- public void write(char cbuf[], int off, int len) throws IOException {
- int space = buf.length - pos;
- if (len < space) {
- System.arraycopy(cbuf, off, buf, pos, len);
- pos += len;
- } else if (len<BUFSIZE) {
- // if the data to write is small enough, buffer it.
- System.arraycopy(cbuf, off, buf, pos, space);
+ public void write(char arr[], int off, int len) throws IOException {
+ for(;;) {
+ int space = buf.length - pos;
+
+ if (len <= space) {
+ System.arraycopy(arr, off, buf, pos, len);
+ pos += len;
+ return;
+ } else if (len > buf.length) {
+ if (pos>0) {
+ flush(buf,0,pos); // flush
+ pos=0;
+ }
+ // don't buffer, just write to sink
+ flush(arr, off, len);
+ return;
+ }
+
+ // buffer is too big to fit in the free space, but
+ // not big enough to warrant writing on its own.
+ // write whatever we can fit, then flush and iterate.
+
+ System.arraycopy(arr, off, buf, pos, space);
flush(buf, 0, buf.length);
- pos = len-space;
- System.arraycopy(cbuf, off+space, buf, 0, pos);
- } else {
- flush(buf,0,pos); // flush
- pos=0;
- // don't buffer, just write to sink
- flush(cbuf, off, len);
+ pos = 0;
+ off += space;
+ len -= space;
}
}
@Override
public void write(String str, int off, int len) throws IOException {
- int space = buf.length - pos;
- if (len < space) {
- str.getChars(off, off+len, buf, pos);
- pos += len;
- } else if (len<BUFSIZE) {
- // if the data to write is small enough, buffer it.
+ for(;;) {
+ int space = buf.length - pos;
+
+ if (len <= space) {
+ str.getChars(off, off+len, buf, pos);
+ pos += len;
+ return;
+ } else if (len > buf.length) {
+ if (pos>0) {
+ flush(buf,0,pos); // flush
+ pos=0;
+ }
+ // don't buffer, just write to sink
+ flush(str, off, len);
+ return;
+ }
+
+ // buffer is too big to fit in the free space, but
+ // not big enough to warrant writing on its own.
+ // write whatever we can fit, then flush and iterate.
+
str.getChars(off, off+space, buf, pos);
flush(buf, 0, buf.length);
- str.getChars(off+space, off+len, buf, 0);
- pos = len-space;
- } else {
- flush(buf,0,pos); // flush
- pos=0;
- // don't buffer, just write to sink
- flush(str, off, len);
+ pos = 0;
+ off += space;
+ len -= space;
}
}
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/util/FastOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/util/FastOutputStream.java?rev=1374481&r1=1374480&r2=1374481&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/util/FastOutputStream.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/util/FastOutputStream.java Sat Aug 18 00:17:09 2012
@@ -57,8 +57,8 @@ public class FastOutputStream extends Ou
public void write(byte b) throws IOException {
if (pos >= buf.length) {
- flush(buf, 0, buf.length);
written += pos;
+ flush(buf, 0, buf.length);
pos=0;
}
buf[pos++] = b;
@@ -66,29 +66,40 @@ public class FastOutputStream extends Ou
@Override
public void write(byte arr[], int off, int len) throws IOException {
- int space = buf.length - pos;
- if (len < space) {
- System.arraycopy(arr, off, buf, pos, len);
- pos += len;
- } else if (len<buf.length) {
- // if the data to write is small enough, buffer it.
+
+ for(;;) {
+ int space = buf.length - pos;
+
+ if (len <= space) {
+ System.arraycopy(arr, off, buf, pos, len);
+ pos += len;
+ return;
+ } else if (len > buf.length) {
+ if (pos>0) {
+ flush(buf,0,pos); // flush
+ written += pos;
+ pos=0;
+ }
+ // don't buffer, just write to sink
+ flush(arr, off, len);
+ written += len;
+ return;
+ }
+
+ // buffer is too big to fit in the free space, but
+ // not big enough to warrant writing on its own.
+ // write whatever we can fit, then flush and iterate.
+
System.arraycopy(arr, off, buf, pos, space);
+ written += buf.length; // important to do this first, since buf.length can change after a flush!
flush(buf, 0, buf.length);
- written += buf.length;
- pos = len-space;
- System.arraycopy(arr, off+space, buf, 0, pos);
- } else {
- if (pos>0) {
- flush(buf,0,pos); // flush
- written += pos;
- pos=0;
- }
- // don't buffer, just write to sink
- flush(arr, off, len);
- written += len;
+ pos = 0;
+ off += space;
+ len -= space;
}
}
+
/** reserve at least len bytes at the end of the buffer.
* Invalid if len > buffer.length
* @param len
@@ -182,8 +193,8 @@ public class FastOutputStream extends Ou
*/
public void flushBuffer() throws IOException {
if (pos > 0) {
- flush(buf, 0, pos);
written += pos;
+ flush(buf, 0, pos);
pos=0;
}
}