You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by gg...@apache.org on 2023/11/12 13:56:27 UTC
(commons-compress) branch master updated: COMPRESS-650 lz4 index out of bounds (#437)
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-compress.git
The following commit(s) were added to refs/heads/master by this push:
new 943521f5a COMPRESS-650 lz4 index out of bounds (#437)
943521f5a is described below
commit 943521f5a5eb40d399c97dc99d90c3674850ddb6
Author: cpreisler <ch...@gmail.com>
AuthorDate: Sun Nov 12 07:56:22 2023 -0600
COMPRESS-650 lz4 index out of bounds (#437)
* Fixed write so it no longer throws an IndexOutOfBoundsException when data is larger than Block size. Test was added.
* Fixed so that writing byte by byte produces same result as bulk write. Tests added.
* Created FramedLZ4CompressorOutputStreamTest and moved framed tests out of block tests.
* Removed comment.
* reverting changes.
* removed unused count variable.
* Removed comment and useless for loop.
* Changed to match master branch.
* removed some spaces.
---------
Co-authored-by: Chad Preisler <42...@users.noreply.gitlab.com>
---
.../lz4/FramedLZ4CompressorOutputStream.java | 26 ++++++---
.../lz4/FramedLZ4CompressorOutputStreamTest.java | 66 ++++++++++++++++++++++
2 files changed, 84 insertions(+), 8 deletions(-)
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
index 612b0a50a..6845f5466 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
@@ -168,6 +168,7 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream {
private final byte[] blockDependencyBuffer;
private int collectedBlockDependencyBytes;
+ private int currentIndex;
/**
* Constructs a new output stream that compresses data using the
@@ -223,18 +224,22 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream {
}
/**
- * Compresses all remaining data and writes it to the stream,
- * doesn't close the underlying stream.
+ * Compresses all blockDataRemaining data and writes it to the stream,
+ doesn't close the underlying stream.
* @throws IOException if an error occurs
*/
public void finish() throws IOException {
if (!finished) {
+ flushBlock();
writeTrailer();
finished = true;
}
}
- private void flushBlock(int currentIndex) throws IOException {
+ private void flushBlock() throws IOException {
+ if (currentIndex == 0) {
+ return;
+ }
final boolean withBlockDependency = params.withBlockDependency;
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (BlockLZ4CompressorOutputStream o = new BlockLZ4CompressorOutputStream(baos, params.lz77params)) {
@@ -274,13 +279,18 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream {
if (params.withContentChecksum) {
contentHash.update(data, off, len);
}
- final int blockDataLength = blockData.length;
+ int blockDataRemaining = blockData.length - currentIndex;
while (len > 0) {
- int copyLen = Math.min(len, blockDataLength);
- System.arraycopy(data, off, blockData, 0, copyLen);
- off += blockDataLength;
+ int copyLen = Math.min(len, blockDataRemaining);
+ System.arraycopy(data, off, blockData, currentIndex, copyLen);
+ off += copyLen;
+ blockDataRemaining -= copyLen;
len -= copyLen;
- flushBlock(copyLen);
+ currentIndex += copyLen;
+ if (blockDataRemaining == 0) {
+ flushBlock();
+ blockDataRemaining = blockData.length;
+ }
}
}
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStreamTest.java
new file mode 100644
index 000000000..97fee10a6
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStreamTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress.compressors.lz4;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import org.junit.jupiter.api.Test;
+
+public class FramedLZ4CompressorOutputStreamTest {
+
+ @Test
+ public void testWriteByteArrayVsWriteByte() throws IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ StringBuilder sb = new StringBuilder();
+ sb.append("abcdefghijklmnop");
+ byte[] random = sb.toString().getBytes();
+ try (FramedLZ4CompressorOutputStream compressor =
+ new FramedLZ4CompressorOutputStream(buffer,
+ new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K64, true, false, false))) {
+ compressor.write(random);
+ compressor.finish();
+ }
+ byte[] bulkOutput = buffer.toByteArray();
+ buffer = new ByteArrayOutputStream();
+ try (FramedLZ4CompressorOutputStream compressor =
+ new FramedLZ4CompressorOutputStream(buffer,
+ new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K64, true, false, false))) {
+ for (int i = 0; i < random.length; i++) {
+ compressor.write(random[i]);
+ }
+ compressor.finish();
+ }
+ byte[] singleOutput = buffer.toByteArray();
+ assertTrue(Arrays.equals(bulkOutput, singleOutput));
+ }
+
+ @Test
+ public void testFinishWithNoWrite() throws IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ try (FramedLZ4CompressorOutputStream compressor =
+ new FramedLZ4CompressorOutputStream(buffer,
+ new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K64, true, false, false))) {
+ // do nothing here. this will test that flush on close doesn't throw any exceptions if no data is written.
+ }
+ assertTrue(buffer.size() == 15, "only the trailer gets written.");
+ }
+
+}