You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by gg...@apache.org on 2025/05/22 15:01:20 UTC
(commons-io) branch master updated: QueueInputStream reads all but the first byte without waiting. (#748)
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-io.git
The following commit(s) were added to refs/heads/master by this push:
new fd10fed8f QueueInputStream reads all but the first byte without waiting. (#748)
fd10fed8f is described below
commit fd10fed8ffd816f8e52fba0328f3522693cdd2e1
Author: maxxedev <50...@users.noreply.github.com>
AuthorDate: Thu May 22 08:01:15 2025 -0700
QueueInputStream reads all but the first byte without waiting. (#748)
* QueueInputStream reads all but the first byte without waiting.
Fix so that bulk reads avoid getting stuck if a timeout is set
and at least one byte is available.
* Call BlockingQueue::drainTo for improved performance
* Refactor bulk read
* Add benchmark for `QueueInputStream/QueueOutputStream`
Adds a small benchmark to measure how much time does it take to transfer
1 MiB from a `QueueOutputStream` to a `QueueInputStream`.
* More optimizations on bulk read
* Improve test coverage
* Add missing Javadoc since tag
* Use final
Reduce vertical whitespace
* Use the exact same invariant checks as the JDK superclass
Add missing Javadoc `@param` tag
* Update src/main/java/org/apache/commons/io/input/QueueInputStream.java
Co-authored-by: Piotr P. Karwasz <pi...@github.copernik.eu>
* Update src/main/java/org/apache/commons/io/input/QueueInputStream.java
* improve tests
* improve test
---------
Co-authored-by: Piotr P. Karwasz <pk...@apache.org>
Co-authored-by: Gary Gregory <ga...@users.noreply.github.com>
Co-authored-by: Piotr P. Karwasz <pi...@github.copernik.eu>
---
src/changes/changes.xml | 1 +
.../apache/commons/io/input/QueueInputStream.java | 47 ++++++++
.../commons/io/input/QueueInputStreamTest.java | 127 ++++++++++++++++++++-
.../commons/io/jmh/QueueStreamBenchmark.java | 76 ++++++++++++
4 files changed, 250 insertions(+), 1 deletion(-)
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index 835750117..177a2f6b1 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -47,6 +47,7 @@ The <action> type attribute can be add,update,fix,remove.
<body>
<release version="2.20.0" date="YYYY-MM-DD" description="Version 2.19.1: Java 8 or later is required.">
<!-- FIX -->
+ <action dev="maxxedev" type="fix" due-to="maxxedev">QueueInputStream reads all but the first byte without waiting</action>
<action dev="ggregory" type="fix" due-to="Jesse Glick">[javadoc] Rename parameter of ProxyOutputStream.write(int) #740.</action>
<action dev="ggregory" type="fix" issue="IO-875" due-to="Pierre Baumard, Gary Gregory">CopyDirectoryVisitor ignores fileFilter #743.</action>
<action dev="ggregory" type="fix" due-to="Gary Gregory">org.apache.commons.io.build.AbstractOrigin.getReader(Charset) now maps a null Charset to the default Charset.</action>
diff --git a/src/main/java/org/apache/commons/io/input/QueueInputStream.java b/src/main/java/org/apache/commons/io/input/QueueInputStream.java
index cc72bc29d..a6f3699e2 100644
--- a/src/main/java/org/apache/commons/io/input/QueueInputStream.java
+++ b/src/main/java/org/apache/commons/io/input/QueueInputStream.java
@@ -23,6 +23,8 @@
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Objects;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@@ -224,4 +226,49 @@ public int read() {
}
}
+ /**
+ * Reads up to {@code length} bytes of data from the input stream into
+ * an array of bytes. The first byte is read while honoring the timeout; the rest are read while <i>not</i> honoring
+ * the timeout. The number of bytes actually read is returned as an integer.
+ *
+ * @param b the buffer into which the data is read.
+ * @param offset the start offset in array {@code b} at which the data is written.
+ * @param length the maximum number of bytes to read.
+ * @return the total number of bytes read into the buffer, or {@code -1} if there is no more data because the
+ * end of the stream has been reached.
+ * @throws NullPointerException If {@code b} is {@code null}.
+ * @throws IllegalStateException if thread is interrupted while waiting for the first byte.
+ * @throws IndexOutOfBoundsException if {@code offset} is negative, {@code length} is negative, or {@code length} is
+ * greater than {@code b.length - offset}.
+ * @since 2.20.0
+ */
+ @Override
+ public int read(final byte[] b, final int offset, final int length) {
+ if (b == null) {
+ throw new NullPointerException();
+ } else if (offset < 0 || length < 0 || length > b.length - offset) {
+ throw new IndexOutOfBoundsException(
+ String.format("Range [%d, %<d + %d) out of bounds for length %d", offset, length, b.length));
+ } else if (length == 0) {
+ return 0;
+ }
+ final List<Integer> drain = new ArrayList<>(Math.min(length, blockingQueue.size()));
+ blockingQueue.drainTo(drain, length);
+ if (drain.isEmpty()) {
+ // no data immediately available. wait for first byte
+ final int value = read();
+ if (value == EOF) {
+ return EOF;
+ }
+ drain.add(value);
+ blockingQueue.drainTo(drain, length - 1);
+ }
+ int i = 0;
+ for (final Integer value : drain) {
+ b[offset + i] = (byte) (0xFF & value);
+ i++;
+ }
+ return i;
+ }
+
}
diff --git a/src/test/java/org/apache/commons/io/input/QueueInputStreamTest.java b/src/test/java/org/apache/commons/io/input/QueueInputStreamTest.java
index 742698b5f..267480e25 100644
--- a/src/test/java/org/apache/commons/io/input/QueueInputStreamTest.java
+++ b/src/test/java/org/apache/commons/io/input/QueueInputStreamTest.java
@@ -16,19 +16,30 @@
*/
package org.apache.commons.io.input;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.commons.lang3.ArrayUtils.EMPTY_BYTE_ARRAY;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTimeout;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+import static org.junit.jupiter.api.DynamicTest.dynamicTest;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.time.Duration;
import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
@@ -40,7 +51,9 @@
import org.apache.commons.io.output.QueueOutputStreamTest;
import org.apache.commons.lang3.StringUtils;
import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.DynamicTest;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestFactory;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
@@ -124,13 +137,125 @@ public void testAvailableAfterOpen(final String inputData) throws IOException {
public void testBufferedReads(final String inputData) throws IOException {
final BlockingQueue<Integer> queue = new LinkedBlockingQueue<>();
try (BufferedInputStream inputStream = new BufferedInputStream(new QueueInputStream(queue));
- QueueOutputStream outputStream = new QueueOutputStream(queue)) {
+ QueueOutputStream outputStream = new QueueOutputStream(queue)) {
outputStream.write(inputData.getBytes(StandardCharsets.UTF_8));
final String actualData = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
assertEquals(inputData, actualData);
}
}
+ @ParameterizedTest(name = "inputData={0}")
+ @MethodSource("inputData")
+ public void testReadLineByLineQueue(final String inputData) throws IOException {
+ final String[] lines = inputData.split("\n");
+ final BlockingQueue<Integer> queue = new LinkedBlockingQueue<>();
+ try (QueueInputStream inputStream = QueueInputStream.builder()
+ .setBlockingQueue(queue)
+ .setTimeout(Duration.ofHours(1))
+ .get();
+ QueueOutputStream outputStream = inputStream.newQueueOutputStream()) {
+
+ doTestReadLineByLine(inputData, inputStream, outputStream);
+ }
+ }
+
+ @ParameterizedTest(name = "inputData={0}")
+ @MethodSource("inputData")
+ public void testReadLineByLineFile(final String inputData) throws IOException {
+ final Path tempFile = Files.createTempFile(getClass().getSimpleName(), ".txt");
+ try (InputStream inputStream = Files.newInputStream(tempFile);
+ OutputStream outputStream = Files.newOutputStream(tempFile)) {
+
+ doTestReadLineByLine(inputData, inputStream, outputStream);
+ } finally {
+ Files.delete(tempFile);
+ }
+ }
+
+ private void doTestReadLineByLine(final String inputData, final InputStream inputStream, final OutputStream outputStream) throws IOException {
+ final String[] lines = inputData.split("\n");
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, UTF_8))) {
+ for (String line : lines) {
+ outputStream.write(line.getBytes(UTF_8));
+ outputStream.write('\n');
+
+ final String actualLine = reader.readLine();
+ assertEquals(line, actualLine);
+ }
+ }
+ }
+
+ @TestFactory
+ public DynamicTest[] bulkReadErrorHandlingTests() {
+ final QueueInputStream queueInputStream = new QueueInputStream();
+ return new DynamicTest[] {
+ dynamicTest("Offset too big", () ->
+ assertThrows(IndexOutOfBoundsException.class, () ->
+ queueInputStream.read(EMPTY_BYTE_ARRAY, 1, 0))),
+
+ dynamicTest("Offset negative", () ->
+ assertThrows(IndexOutOfBoundsException.class, () ->
+ queueInputStream.read(EMPTY_BYTE_ARRAY, -1, 0))),
+
+ dynamicTest("Length too big", () ->
+ assertThrows(IndexOutOfBoundsException.class, () ->
+ queueInputStream.read(EMPTY_BYTE_ARRAY, 0, 1))),
+
+ dynamicTest("Length negative", () ->
+ assertThrows(IndexOutOfBoundsException.class, () ->
+ queueInputStream.read(EMPTY_BYTE_ARRAY, 0, -1))),
+ };
+ }
+
+ @Test
+ public void testBulkReadZeroLength() {
+ final QueueInputStream queueInputStream = new QueueInputStream();
+ final int read = queueInputStream.read(EMPTY_BYTE_ARRAY, 0, 0);
+ assertEquals(0, read);
+ }
+
+ @ParameterizedTest(name = "inputData={0}")
+ @MethodSource("inputData")
+ public void testBulkReadWaiting(final String inputData) throws IOException {
+ assumeFalse(inputData.isEmpty());
+
+ final CountDownLatch onPollLatch = new CountDownLatch(1);
+ final CountDownLatch afterWriteLatch = new CountDownLatch(1);
+ final LinkedBlockingQueue<Integer> queue = new LinkedBlockingQueue<Integer>() {
+ @Override
+ public Integer poll(final long timeout, final TimeUnit unit) throws InterruptedException {
+ onPollLatch.countDown();
+ afterWriteLatch.await(1, TimeUnit.HOURS);
+ return super.poll(timeout, unit);
+ }
+ };
+
+ // Simulate scenario where there is not data immediately available when bulk reading and QueueInputStream has to
+ // wait.
+ try (QueueInputStream queueInputStream = QueueInputStream.builder()
+ .setBlockingQueue(queue)
+ .setTimeout(Duration.ofHours(1))
+ .get()) {
+ final QueueOutputStream queueOutputStream = queueInputStream.newQueueOutputStream();
+ final CompletableFuture<Void> future = CompletableFuture.runAsync(() -> {
+ try {
+ onPollLatch.await(1, TimeUnit.HOURS);
+ queueOutputStream.write(inputData.getBytes(UTF_8));
+ afterWriteLatch.countDown();
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ final byte[] data = new byte[inputData.length()];
+ final int read = queueInputStream.read(data, 0, data.length);
+ assertEquals(inputData.length(), read);
+ final String outputData = new String(data, 0, read, StandardCharsets.UTF_8);
+ assertEquals(inputData, outputData);
+ assertDoesNotThrow(() -> future.get());
+ }
+ }
+
@ParameterizedTest(name = "inputData={0}")
@MethodSource("inputData")
public void testBufferedReadWrite(final String inputData) throws IOException {
diff --git a/src/test/java/org/apache/commons/io/jmh/QueueStreamBenchmark.java b/src/test/java/org/apache/commons/io/jmh/QueueStreamBenchmark.java
new file mode 100644
index 000000000..eebcb30a5
--- /dev/null
+++ b/src/test/java/org/apache/commons/io/jmh/QueueStreamBenchmark.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.io.jmh;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.input.QueueInputStream;
+import org.apache.commons.io.output.QueueOutputStream;
+import org.apache.commons.lang3.RandomUtils;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Group;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.infra.Blackhole;
+
+/**
+ * Measures the amount of time to push 1 MiB to a {@link QueueOutputStream} and read it using a {@link QueueInputStream}
+ */
+@BenchmarkMode(Mode.SampleTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Group)
+public class QueueStreamBenchmark {
+
+ private static final int CAPACITY = 1024 * 1024;
+ private static final int BUFFER_SIZE = 1024;
+
+ private final BlockingQueue<Integer> queue = new LinkedBlockingQueue<>(CAPACITY);
+ private final QueueInputStream inputStream = QueueInputStream.builder()
+ .setBlockingQueue(queue)
+ .get();
+ private final QueueOutputStream outputStream = inputStream.newQueueOutputStream();
+
+ private final byte[] input = RandomUtils.insecure().randomBytes(CAPACITY);
+ private final byte[] output = new byte[BUFFER_SIZE];
+
+ @Benchmark
+ @Group("streams")
+ public void output() throws Exception {
+ int sent = 0;
+ while (sent < CAPACITY) {
+ final int len = Math.min(CAPACITY - sent, BUFFER_SIZE);
+ outputStream.write(input, sent, len);
+ sent += len;
+ }
+ }
+
+ @Benchmark
+ @Group("streams")
+ public void input(Blackhole bh) throws Exception {
+ int received = 0;
+ while (received < CAPACITY) {
+ final int len = inputStream.read(output, 0, BUFFER_SIZE);
+ bh.consume(output);
+ received += len;
+ }
+ }
+}