You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by se...@apache.org on 2013/06/04 02:03:06 UTC
svn commit: r1489238 - in /commons/proper/io/trunk/src: changes/changes.xml
main/java/org/apache/commons/io/IOUtils.java
main/java/org/apache/commons/io/output/ChunkedOutputStream.java
main/java/org/apache/commons/io/output/ChunkedWriter.java
Author: sebb
Date: Tue Jun 4 00:03:06 2013
New Revision: 1489238
URL: http://svn.apache.org/r1489238
Log:
IO-382 Chunked IO for large arrays
Added:
commons/proper/io/trunk/src/main/java/org/apache/commons/io/output/ChunkedOutputStream.java (with props)
commons/proper/io/trunk/src/main/java/org/apache/commons/io/output/ChunkedWriter.java (with props)
Modified:
commons/proper/io/trunk/src/changes/changes.xml
commons/proper/io/trunk/src/main/java/org/apache/commons/io/IOUtils.java
Modified: commons/proper/io/trunk/src/changes/changes.xml
URL: http://svn.apache.org/viewvc/commons/proper/io/trunk/src/changes/changes.xml?rev=1489238&r1=1489237&r2=1489238&view=diff
==============================================================================
--- commons/proper/io/trunk/src/changes/changes.xml (original)
+++ commons/proper/io/trunk/src/changes/changes.xml Tue Jun 4 00:03:06 2013
@@ -47,6 +47,11 @@ The <action> type attribute can be add,u
<body>
<!-- The release date is the date RC is cut -->
<release version="2.5" date="2013-??-??" description="New features and bug fixes.">
+ <action issue="IO-382" dev="sebb" type="add">
+ Chunked IO for large arrays.
+ Added writeChunked(byte[], OutputStream) and writeChunked(char[] Writer)
+ Added ChunkedOutputStream, ChunkedWriter
+ </action>
<action issue="IO-385" dev="sebb" type="fix">
FileUtils.doCopyFile can potentially loop for ever
Exit loop if no data to copy
Modified: commons/proper/io/trunk/src/main/java/org/apache/commons/io/IOUtils.java
URL: http://svn.apache.org/viewvc/commons/proper/io/trunk/src/main/java/org/apache/commons/io/IOUtils.java?rev=1489238&r1=1489237&r2=1489238&view=diff
==============================================================================
--- commons/proper/io/trunk/src/main/java/org/apache/commons/io/IOUtils.java (original)
+++ commons/proper/io/trunk/src/main/java/org/apache/commons/io/IOUtils.java Tue Jun 4 00:03:06 2013
@@ -1340,6 +1340,32 @@ public class IOUtils {
}
/**
+ * Writes bytes from a <code>byte[]</code> to an <code>OutputStream</code> using chunked writes.
+ * This is intended for writing very large byte arrays which might otherwise cause excessive
+ * memory usage if the native code has to allocate a copy.
+ *
+ * @param data the byte array to write, do not modify during output,
+ * null ignored
+ * @param output the <code>OutputStream</code> to write to
+ * @throws NullPointerException if output is null
+ * @throws IOException if an I/O error occurs
+ * @since 2.5
+ */
+ public static void writeChunked(final byte[] data, final OutputStream output)
+ throws IOException {
+ if (data != null) {
+ int bytes = data.length;
+ int offset = 0;
+ while(bytes > 0) {
+ int chunk = Math.min(bytes, DEFAULT_BUFFER_SIZE);
+ output.write(data, offset, chunk);
+ bytes -= chunk;
+ offset += chunk;
+ }
+ }
+ }
+
+ /**
* Writes bytes from a <code>byte[]</code> to chars on a <code>Writer</code>
* using the default character encoding of the platform.
* <p>
@@ -1421,6 +1447,31 @@ public class IOUtils {
}
/**
+ * Writes chars from a <code>char[]</code> to a <code>Writer</code> using chunked writes.
+ * This is intended for writing very large byte arrays which might otherwise cause excessive
+ * memory usage if the native code has to allocate a copy.
+ *
+ * @param data the char array to write, do not modify during output,
+ * null ignored
+ * @param output the <code>Writer</code> to write to
+ * @throws NullPointerException if output is null
+ * @throws IOException if an I/O error occurs
+ * @since 2.5
+ */
+ public static void writeChunked(final char[] data, final Writer output) throws IOException {
+ if (data != null) {
+ int bytes = data.length;
+ int offset = 0;
+ while(bytes > 0) {
+ int chunk = Math.min(bytes, DEFAULT_BUFFER_SIZE);
+ output.write(data, offset, chunk);
+ bytes -= chunk;
+ offset += chunk;
+ }
+ }
+ }
+
+ /**
* Writes chars from a <code>char[]</code> to bytes on an
* <code>OutputStream</code>.
* <p>
@@ -2825,4 +2876,5 @@ public class IOUtils {
throw new EOFException("Length to read: " + expected + " actual: " + actual);
}
}
+
}
Added: commons/proper/io/trunk/src/main/java/org/apache/commons/io/output/ChunkedOutputStream.java
URL: http://svn.apache.org/viewvc/commons/proper/io/trunk/src/main/java/org/apache/commons/io/output/ChunkedOutputStream.java?rev=1489238&view=auto
==============================================================================
--- commons/proper/io/trunk/src/main/java/org/apache/commons/io/output/ChunkedOutputStream.java (added)
+++ commons/proper/io/trunk/src/main/java/org/apache/commons/io/output/ChunkedOutputStream.java Tue Jun 4 00:03:06 2013
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.io.output;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * OutputStream which breaks larger output blocks into chunks.
+ * Native code may need to copy the input array; if the write buffer
+ * is very large this can cause OOME.
+ *
+ * @since 2.5
+ */
+public class ChunkedOutputStream extends FilterOutputStream {
+
+ /**
+ * The default chunk size to use, i.e. {@value} bytes.
+ */
+ private static final int DEFAULT_CHUNK_SIZE = 1024 * 4;
+
+ /**
+ * The maximum chunk size to us when writing data arrays
+ */
+ private final int chunkSize;
+
+ /**
+ * Creates a new stream that uses the specified chunk size.
+ *
+ * @param stream the stream to wrap
+ * @param chunkSize the chunk size to use; must be a positive number.
+ * @throws IllegalArgumentException if the chunk size is <= 0
+ */
+ public ChunkedOutputStream(final OutputStream stream, int chunkSize) {
+ super(stream);
+ if (chunkSize <= 0) {
+ throw new IllegalArgumentException();
+ }
+ this.chunkSize = chunkSize;
+ }
+
+ /**
+ * Creates a new stream that uses a chunk size of {@link #DEFAULT_CHUNK_SIZE}
+ * @param stream the stream to wrap
+ */
+ public ChunkedOutputStream(final OutputStream stream) {
+ this(stream, DEFAULT_CHUNK_SIZE);
+ }
+
+ /**
+ * writes the data buffer in chunks to the underlying stream
+ */
+ @Override
+ public void write(byte[] data, int srcOffset, int length) throws IOException {
+ int bytes = length;
+ int dstOffset = srcOffset;
+ while(bytes > 0) {
+ int chunk = Math.min(bytes, chunkSize);
+ out.write(data, dstOffset, chunk);
+ bytes -= chunk;
+ dstOffset += chunk;
+ }
+ }
+
+}
Propchange: commons/proper/io/trunk/src/main/java/org/apache/commons/io/output/ChunkedOutputStream.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: commons/proper/io/trunk/src/main/java/org/apache/commons/io/output/ChunkedOutputStream.java
------------------------------------------------------------------------------
svn:keywords = Author Date Id Revision
Added: commons/proper/io/trunk/src/main/java/org/apache/commons/io/output/ChunkedWriter.java
URL: http://svn.apache.org/viewvc/commons/proper/io/trunk/src/main/java/org/apache/commons/io/output/ChunkedWriter.java?rev=1489238&view=auto
==============================================================================
--- commons/proper/io/trunk/src/main/java/org/apache/commons/io/output/ChunkedWriter.java (added)
+++ commons/proper/io/trunk/src/main/java/org/apache/commons/io/output/ChunkedWriter.java Tue Jun 4 00:03:06 2013
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.io.output;
+
+import java.io.FilterWriter;
+import java.io.IOException;
+import java.io.Writer;
+
+/**
+ * OutputStream which breaks larger output blocks into chunks.
+ * Native code may need to copy the input array; if the write buffer
+ * is very large this can cause OOME.
+ *
+ * @since 2.5
+ */
+public class ChunkedWriter extends FilterWriter {
+
+ /**
+ * The default chunk size to use, i.e. {@value} bytes.
+ */
+ private static final int DEFAULT_CHUNK_SIZE = 1024 * 4;
+
+ /**
+ * The maximum chunk size to us when writing data arrays
+ */
+ private final int chunkSize;
+
+ /**
+ * Creates a new writer that uses the specified chunk size.
+ *
+ * @param writer the writer to wrap
+ * @param chunkSize the chunk size to use; must be a positive number.
+ * @throws IllegalArgumentException if the chunk size is <= 0
+ */
+ public ChunkedWriter(final Writer writer, int chunkSize) {
+ super(writer);
+ if (chunkSize <= 0) {
+ throw new IllegalArgumentException();
+ }
+ this.chunkSize = chunkSize;
+ }
+
+ /**
+ * Creates a new writer that uses a chunk size of {@link #DEFAULT_CHUNK_SIZE}
+ * @param writer the writer to wrap
+ */
+ public ChunkedWriter(final Writer writer) {
+ this(writer, DEFAULT_CHUNK_SIZE);
+ }
+
+ /**
+ * writes the data buffer in chunks to the underlying writer
+ */
+ @Override
+ public void write(char[] data, int srcOffset, int length) throws IOException {
+ int bytes = length;
+ int dstOffset = srcOffset;
+ while(bytes > 0) {
+ int chunk = Math.min(bytes, chunkSize);
+ out.write(data, dstOffset, chunk);
+ bytes -= chunk;
+ dstOffset += chunk;
+ }
+ }
+
+}
Propchange: commons/proper/io/trunk/src/main/java/org/apache/commons/io/output/ChunkedWriter.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: commons/proper/io/trunk/src/main/java/org/apache/commons/io/output/ChunkedWriter.java
------------------------------------------------------------------------------
svn:keywords = Author Date Id Revision