You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by pk...@apache.org on 2025/09/04 21:18:40 UTC
(commons-io) 01/01: feat: Add incremental `toByteArray` method
This is an automated email from the ASF dual-hosted git repository.
pkarwasz pushed a commit to branch feat/incremental-reading
in repository https://gitbox.apache.org/repos/asf/commons-io.git
commit debe34abe1e9fb522cd94babb6c98a055f2e4b5a
Author: Piotr P. Karwasz <pk...@apache.org>
AuthorDate: Thu Sep 4 23:13:32 2025 +0200
feat: Add incremental `toByteArray` method
This introduces `toByteArray(InputStream input, int size, int bufferSize)`, which reads the stream in chunks of `bufferSize` instead of allocating the full array up front.
By reading incrementally, the method:
* Validates that the stream actually contains `size` bytes before completing the allocation.
* Prevents excessive memory usage if a corrupted or malicious `size` value is provided.
* Offers safer handling for untrusted input compared to the direct-allocation variant.
---
src/changes/changes.xml | 1 +
src/main/java/org/apache/commons/io/IOUtils.java | 129 ++++++++++++++++-----
.../org/apache/commons/io/RandomAccessFiles.java | 4 +
.../java/org/apache/commons/io/IOUtilsTest.java | 45 +++++++
4 files changed, 152 insertions(+), 27 deletions(-)
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index de92deabf..a10b46538 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -57,6 +57,7 @@ The <action> type attribute can be add,update,fix,remove.
<action dev="ggregory" type="add" due-to="Gary Gregory">Add org.apache.commons.io.output.ProxyOutputStream.writeRepeat(byte[], int, int, long).</action>
<action dev="ggregory" type="add" due-to="Gary Gregory">Add org.apache.commons.io.output.ProxyOutputStream.writeRepeat(byte[], long).</action>
<action dev="ggregory" type="add" due-to="Gary Gregory">Add org.apache.commons.io.output.ProxyOutputStream.writeRepeat(int, long).</action>
+ <action dev="pkarwasz" type="add" due-to="Piotr P. Karwasz">Added toByteArray(InputStream, int, int) for safer incremental reading with size validation.</action>
<!-- UPDATE -->
<action type="update" dev="ggregory" due-to="Gary Gregory, Dependabot">Bump org.apache.commons:commons-parent from 85 to 87 #774.</action>
<action type="update" dev="ggregory" due-to="Gary Gregory">[test] Bump commons-codec:commons-codec from 1.18.0 to 1.19.0.</action>
diff --git a/src/main/java/org/apache/commons/io/IOUtils.java b/src/main/java/org/apache/commons/io/IOUtils.java
index cd8d042a9..96c6e6824 100644
--- a/src/main/java/org/apache/commons/io/IOUtils.java
+++ b/src/main/java/org/apache/commons/io/IOUtils.java
@@ -65,6 +65,7 @@
import org.apache.commons.io.function.IOConsumer;
import org.apache.commons.io.function.IOSupplier;
import org.apache.commons.io.function.IOTriFunction;
+import org.apache.commons.io.input.BoundedInputStream;
import org.apache.commons.io.input.CharSequenceReader;
import org.apache.commons.io.input.QueueInputStream;
import org.apache.commons.io.output.AppendableWriter;
@@ -2659,37 +2660,60 @@ public static byte[] toByteArray(final InputStream inputStream) throws IOExcepti
}
/**
- * Gets the contents of an {@link InputStream} as a {@code byte[]}. Use this method instead of
- * {@link #toByteArray(InputStream)} when {@link InputStream} size is known.
+ * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}.
*
- * @param input the {@link InputStream} to read.
- * @param size the size of {@link InputStream} to read, where 0 < {@code size} <= length of input stream.
- * @return byte [] of length {@code size}.
- * @throws IOException if an I/O error occurs or {@link InputStream} length is smaller than parameter {@code size}.
- * @throws IllegalArgumentException if {@code size} is less than zero.
+ * <p>
+ * This variant allocates the target array immediately and attempts to fill it in one pass.
+ * It assumes that {@code size} is correct.
+ * If the stream ends prematurely, an {@link EOFException} is thrown.
+ * </p>
+ *
+ * <p>
+ * <strong>Important:</strong> This method does <em>not</em> defend against corrupted
+ * or untrusted {@code size} values.
+ * For untrusted input, use {@link #toByteArray(InputStream, int, int)} instead,
+ * which validates that the stream contains at least {@code size} bytes before allocating the target array.
+ * </p>
+ *
+ * @param input the {@link InputStream} to read; must not be {@code null}.
+ * @param size the exact number of bytes to read; must be {@code >= 0}.
+ * @return a new byte array of length {@code size}.
+ * @throws IllegalArgumentException if {@code size} is negative.
+ * @throws EOFException if the stream ends before {@code size} bytes are read.
+ * @throws IOException if an I/O error occurs while reading.
+ * @throws NullPointerException if {@code input} is {@code null}.
* @since 2.1
*/
public static byte[] toByteArray(final InputStream input, final int size) throws IOException {
- if (size == 0) {
- return EMPTY_BYTE_ARRAY;
+ Objects.requireNonNull(input, "input");
+ if (size < 0) {
+ throw new IllegalArgumentException("Size must be equal or greater than zero: " + size);
}
- return toByteArray(Objects.requireNonNull(input, "input")::read, size);
+ return toByteArray(input::read, size);
}
/**
- * Gets contents of an {@link InputStream} as a {@code byte[]}.
- * Use this method instead of {@link #toByteArray(InputStream)}
- * when {@link InputStream} size is known.
- * <strong>NOTE:</strong> the method checks that the length can safely be cast to an int without truncation
- * before using {@link IOUtils#toByteArray(InputStream, int)} to read into the byte array.
- * (Arrays can have no more than Integer.MAX_VALUE entries anyway.)
+ * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}.
*
- * @param input the {@link InputStream} to read.
- * @param size the size of {@link InputStream} to read, where 0 < {@code size} <= min(Integer.MAX_VALUE, length of input stream).
- * @return byte [] the requested byte array, of length {@code size}.
- * @throws IOException if an I/O error occurs or {@link InputStream} length is less than {@code size}.
- * @throws IllegalArgumentException if size is less than zero or size is greater than Integer.MAX_VALUE.
- * @see IOUtils#toByteArray(InputStream, int)
+ * <p>
+ * This is a convenience overload of {@link #toByteArray(InputStream, int, int)} that accepts a
+ * {@code long} size parameter. The value is checked to ensure it does not exceed
+ * {@link Integer#MAX_VALUE} before being safely converted to {@code int}.
+ * </p>
+ *
+ * <p>
+ * All behavior, validation rules, and exceptions are otherwise identical to
+ * {@link #toByteArray(InputStream, int, int)}.
+ * </p>
+ *
+ * @param input the {@link InputStream} to read; must not be {@code null}.
+ * @param size the exact number of bytes to read; must be {@code >= 0} and {@code <= Integer.MAX_VALUE}.
+ * @return a new byte array of length {@code size}.
+ * @throws IllegalArgumentException if {@code size} is negative or greater than {@link Integer#MAX_VALUE}.
+ * @throws EOFException if the stream ends before {@code size} bytes are read.
+ * @throws IOException if an I/O error occurs while reading.
+ * @throws NullPointerException if {@code input} is {@code null}.
+ * @see #toByteArray(InputStream, int, int)
* @since 2.1
*/
public static byte[] toByteArray(final InputStream input, final long size) throws IOException {
@@ -2699,6 +2723,62 @@ public static byte[] toByteArray(final InputStream input, final long size) throw
return toByteArray(input, (int) size);
}
+ /**
+ * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}.
+ *
+ * <p>
+ * This variant validates that the stream actually contains {@code size} bytes.
+ * It is suitable for untrusted input because it prevents oversized allocations when the provided {@code size}
+ * is corrupted or malicious.
+ * </p>
+ *
+ * <ul>
+ * <li>If {@code size <= bufferSize}, the array is allocated directly and filled in a single pass.</li>
+ * <li>
+ * If {@code size > bufferSize}, the stream is read incrementally using a buffer of length {@code bufferSize}.
+ * This avoids allocating an excessively large array up front,
+ * but may temporarily double memory usage due to buffering.
+ * </li>
+ * </ul>
+ *
+ * @param input the {@link InputStream} to read; must not be {@code null}.
+ * @param size the exact number of bytes to read; must be {@code >= 0}.
+ * The actual bytes read are validated to equal {@code size}.
+ * @param bufferSize the buffer size for incremental reading; must be {@code > 0}.
+ * @return a new byte array of length {@code size}.
+ * @throws IllegalArgumentException if {@code size} is negative or {@code bufferSize <= 0}.
+ * @throws EOFException if the stream ends before {@code size} bytes are read.
+ * @throws IOException if an I/O error occurs while reading.
+ * @throws NullPointerException if {@code input} is {@code null}.
+ * @since 2.21.0
+ */
+ public static byte[] toByteArray(final InputStream input, final int size, final int bufferSize) throws IOException {
+ Objects.requireNonNull(input, "input");
+ if (size < 0) {
+ throw new IllegalArgumentException("Size must be equal or greater than zero: " + size);
+ }
+ if (bufferSize <= 0) {
+ throw new IllegalArgumentException("Chunk size must be greater than zero: " + bufferSize);
+ }
+ if (size <= bufferSize) {
+ return toByteArray(input::read, size);
+ }
+ try (UnsynchronizedByteArrayOutputStream output = UnsynchronizedByteArrayOutputStream.builder()
+ .setBufferSize(bufferSize)
+ .get();
+ InputStream boundedInput = BoundedInputStream.builder()
+ .setMaxCount(size)
+ .setPropagateClose(false)
+ .setInputStream(input)
+ .get()) {
+ output.write(boundedInput);
+ if (output.size() != size) {
+ throw new EOFException("Unexpected read size, current: " + output.size() + ", expected: " + size);
+ }
+ return output.toByteArray();
+ }
+ }
+
/**
* Gets the contents of an input as a {@code byte[]}.
*
@@ -2709,11 +2789,6 @@ public static byte[] toByteArray(final InputStream input, final long size) throw
* @throws IllegalArgumentException if {@code size} is less than zero.
*/
static byte[] toByteArray(final IOTriFunction<byte[], Integer, Integer, Integer> input, final int size) throws IOException {
-
- if (size < 0) {
- throw new IllegalArgumentException("Size must be equal or greater than zero: " + size);
- }
-
if (size == 0) {
return EMPTY_BYTE_ARRAY;
}
diff --git a/src/main/java/org/apache/commons/io/RandomAccessFiles.java b/src/main/java/org/apache/commons/io/RandomAccessFiles.java
index 46de0dbf0..d9baa8f73 100644
--- a/src/main/java/org/apache/commons/io/RandomAccessFiles.java
+++ b/src/main/java/org/apache/commons/io/RandomAccessFiles.java
@@ -76,7 +76,11 @@ private static long length(final RandomAccessFile raf) throws IOException {
* other I/O error occurs.
*/
public static byte[] read(final RandomAccessFile input, final long position, final int length) throws IOException {
+ Objects.requireNonNull(input, "input");
input.seek(position);
+ if (length < 0) {
+ throw new IllegalArgumentException("Size must be equal or greater than zero: " + length);
+ }
return IOUtils.toByteArray(input::read, length);
}
diff --git a/src/test/java/org/apache/commons/io/IOUtilsTest.java b/src/test/java/org/apache/commons/io/IOUtilsTest.java
index 56fd1307e..92925bdd7 100644
--- a/src/test/java/org/apache/commons/io/IOUtilsTest.java
+++ b/src/test/java/org/apache/commons/io/IOUtilsTest.java
@@ -90,6 +90,9 @@
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
/**
* This is used to test {@link IOUtils} for correctness. The following checks are performed:
@@ -1659,6 +1662,48 @@ void testToByteArray_InputStream_SizeZero() throws Exception {
}
}
+ @ParameterizedTest
+ @MethodSource
+ void testToByteArray_InputStream_Size_BufferSize_Succeeds(byte[] data, int size, int bufferSize) throws IOException {
+ final ByteArrayInputStream input = new ByteArrayInputStream(data);
+ final byte[] expected = Arrays.copyOf(data, size);
+ final byte[] actual = IOUtils.toByteArray(input, size, bufferSize);
+ assertArrayEquals(expected, actual);
+ }
+
+ private static Stream<Arguments> testToByteArray_InputStream_Size_BufferSize_Succeeds() {
+ final byte[] data = new byte[1024];
+ for (int i = 0; i < 1024; i++) {
+ data[i] = (byte) i;
+ }
+ return Stream.of(
+ // Eager reading
+ Arguments.of(data.clone(), 512, 1024),
+ // Incremental reading
+ Arguments.of(data.clone(), 1024, 512),
+ // No reading
+ Arguments.of(data.clone(), 0, 128));
+ }
+
+ @ParameterizedTest
+ @MethodSource
+ void testToByteArray_InputStream_Size_BufferSize_Throws(
+ int size, int bufferSize, Class<? extends Exception> exceptionClass) throws IOException {
+ try (InputStream input = new NullInputStream(0)) {
+ assertThrows(exceptionClass, () -> IOUtils.toByteArray(input, size, bufferSize));
+ }
+ }
+
+ static Stream<Arguments> testToByteArray_InputStream_Size_BufferSize_Throws() {
+ return Stream.of(
+ // Negative size
+ Arguments.of(-1, 128, IllegalArgumentException.class),
+ // Invalid buffer size
+ Arguments.of(0, 0, IllegalArgumentException.class),
+ // Huge size: should not cause OutOfMemoryError
+ Arguments.of(Integer.MAX_VALUE, 128, EOFException.class));
+ }
+
@Test
void testToByteArray_Reader() throws IOException {
final String charsetName = UTF_8;