You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by gg...@apache.org on 2025/01/01 15:17:47 UTC
(commons-compress) branch master updated: Add GzipCompressorInputStream.Builder allowing to customize the file name and comment Charsets
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-compress.git
The following commit(s) were added to refs/heads/master by this push:
new 314e98a63 Add GzipCompressorInputStream.Builder allowing to customize the file name and comment Charsets
314e98a63 is described below
commit 314e98a63ffc3ed34c68cc11a1fe0db974648fba
Author: Gary Gregory <ga...@gmail.com>
AuthorDate: Wed Jan 1 10:17:41 2025 -0500
Add GzipCompressorInputStream.Builder allowing to customize the file
name and comment Charsets
- Fix JUnit assert message
- Better lambda
- Clarify comments
---
src/changes/changes.xml | 1 +
.../gzip/GzipCompressorInputStream.java | 158 +++++++++++++++------
.../gzip/GzipCompressorOutputStreamTest.java | 22 ++-
3 files changed, 134 insertions(+), 47 deletions(-)
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index 1c8018cb9..d3c5df614 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -71,6 +71,7 @@ The <action> type attribute can be add,update,fix,remove.
<action type="add" dev="ggregory" due-to="Danny Deschenes, Gary Gregory">GzipCompressorInputStream reads the modification time (MTIME) and stores its value incorrectly multiplied by 1,000.</action>
<action type="add" dev="ggregory" due-to="Danny Deschenes, Gary Gregory">GzipCompressorInputStream writes the modification time (MTIME) the value incorrectly divided by 1,000.</action>
<action type="add" dev="ggregory" due-to="Danny Deschenes, Gary Gregory">Add optional FHCRC to GZIP header #627.</action>
+ <action type="add" dev="ggregory" due-to="Gary Gregory">Add GzipCompressorInputStream.Builder allowing to customize the file name and comment Charsets.</action>
<!-- UPDATE -->
<action type="update" dev="ggregory" due-to="Dependabot, Gary Gregory">Bump org.apache.commons:commons-parent from 72 to 78 #563, #567, #574, #582, #587, #595.</action>
<action type="update" dev="ggregory" due-to="Dependabot, Gary Gregory">Bump com.github.luben:zstd-jni from 1.5.6-4 to 1.5.6-8 #565, #578, #601, #616.</action>
diff --git a/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorInputStream.java
index d40b92617..0e8b77b86 100644
--- a/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorInputStream.java
@@ -25,6 +25,7 @@ import java.io.DataInputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.Charset;
import java.util.zip.CRC32;
import java.util.zip.DataFormatException;
import java.util.zip.Deflater;
@@ -33,6 +34,7 @@ import java.util.zip.Inflater;
import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.utils.ByteUtils;
import org.apache.commons.compress.utils.InputStreamStatistics;
+import org.apache.commons.io.build.AbstractStreamBuilder;
import org.apache.commons.io.input.BoundedInputStream;
/**
@@ -66,6 +68,71 @@ import org.apache.commons.io.input.BoundedInputStream;
*/
public class GzipCompressorInputStream extends CompressorInputStream implements InputStreamStatistics {
+ /**
+ * Constructs a new builder of {@link GzipCompressorInputStream}.
+ *
+ * @since 1.28.0
+ */
+ public static class Builder extends AbstractStreamBuilder<GzipCompressorInputStream, Builder> {
+
+ /** True if decompressing multi-member streams. */
+ private boolean decompressConcatenated;
+
+ private Charset fileNameCharset = GzipUtils.GZIP_ENCODING;
+
+ /**
+ * Constructs a new builder of {@link GzipCompressorInputStream}.
+ */
+ public Builder() {
+ // empty
+ }
+
+ @SuppressWarnings("resource") // caller closes
+ @Override
+ public GzipCompressorInputStream get() throws IOException {
+ return new GzipCompressorInputStream(getInputStream(), decompressConcatenated, fileNameCharset);
+ }
+
+ /**
+ * Sets whether we should allow decompressing multiple members.
+ *
+ * @param decompressConcatenated whether we should allow decompressing multiple members.
+ * @return this instance.
+ */
+ public Builder setDecompressConcatenated(final boolean decompressConcatenated) {
+ this.decompressConcatenated = decompressConcatenated;
+ return this;
+ }
+
+ /**
+ * Sets the Charset to use for writing file names and comments, where null maps to {@link GzipUtils#GZIP_ENCODING}.
+ * <p>
+ * <em>Setting a value other than {@link GzipUtils#GZIP_ENCODING} is not compliant with the <a href="https://datatracker.ietf.org/doc/html/rfc1952">RFC
+ * 1952 GZIP File Format Specification</a></em>. Use at your own risk of interoperability issues.
+ * </p>
+ * <p>
+ * The default value is {@link GzipUtils#GZIP_ENCODING}.
+ * </p>
+ *
+ * @param fileNameCharset the Charset to use for writing file names and comments, null maps to {@link GzipUtils#GZIP_ENCODING}.
+ * @return this instance.
+ */
+ public Builder setFileNameCharset(final Charset fileNameCharset) {
+ this.fileNameCharset = fileNameCharset;
+ return this;
+ }
+ }
+
+ /**
+ * Constructs a new builder of {@link GzipCompressorInputStream}.
+ *
+ * @return a new builder of {@link GzipCompressorInputStream}.
+ * @since 1.28.0
+ */
+ public static Builder builder() {
+ return new Builder();
+ }
+
/**
* Checks if the signature matches what is expected for a .gz file.
*
@@ -88,31 +155,35 @@ public class GzipCompressorInputStream extends CompressorInputStream implements
}
}
- private final BoundedInputStream countingStream;
-
- // Compressed input stream, possibly wrapped in a
- // BufferedInputStream, always wrapped in countingStream above
- private final InputStream in;
-
- // True if decompressing multi member streams.
- private final boolean decompressConcatenated;
-
- // Buffer to hold the input data
+ /** Buffer to hold the input data. */
private final byte[] buf = new byte[8192];
- // Amount of data in buf.
+ /** Amount of data in buf. */
private int bufUsed;
- // Decompressor
- private Inflater inf = new Inflater(true);
+ private final BoundedInputStream countingStream;
- // CRC32 from uncompressed data
+ /** CRC32 from uncompressed data. */
private final CRC32 crc = new CRC32();
- // True once everything has been decompressed
+ /** True if decompressing multi-member streams. */
+ private final boolean decompressConcatenated;
+
+ /** True once everything has been decompressed. */
private boolean endReached;
- // used in no-arg read method
+ private final Charset fileNameCharset;
+
+ /**
+ * Compressed input stream, possibly wrapped in a
+ * BufferedInputStream, always wrapped in countingStream above
+ */
+ private final InputStream in;
+
+ /** Decompressor. */
+ private Inflater inflater = new Inflater(true);
+
+ /** Buffer for no-argument read method. */
private final byte[] oneByte = new byte[1];
private final GzipParameters parameters = new GzipParameters();
@@ -121,12 +192,13 @@ public class GzipCompressorInputStream extends CompressorInputStream implements
* Constructs a new input stream that decompresses gzip-compressed data from the specified input stream.
* <p>
* This is equivalent to {@code GzipCompressorInputStream(inputStream, false)} and thus will not decompress concatenated .gz files.
+ * </p>
*
* @param inputStream the InputStream from which this object should be created of
* @throws IOException if the stream could not be created
*/
public GzipCompressorInputStream(final InputStream inputStream) throws IOException {
- this(inputStream, false);
+ this(inputStream, false, GzipUtils.GZIP_ENCODING);
}
/**
@@ -135,22 +207,25 @@ public class GzipCompressorInputStream extends CompressorInputStream implements
* If {@code decompressConcatenated} is {@code false}: This decompressor might read more input than it will actually use. If {@code inputStream} supports
* {@code mark} and {@code reset}, then the input position will be adjusted so that it is right after the last byte of the compressed stream. If
* {@code mark} isn't supported, the input position will be undefined.
+ * </p>
*
* @param inputStream the InputStream from which this object should be created of
* @param decompressConcatenated if true, decompress until the end of the input; if false, stop after the first .gz member
* @throws IOException if the stream could not be created
+ * @deprecated Use {@link Builder#get()}.
*/
+ @Deprecated
public GzipCompressorInputStream(final InputStream inputStream, final boolean decompressConcatenated) throws IOException {
+ this(inputStream, decompressConcatenated, GzipUtils.GZIP_ENCODING);
+ }
+
+ private GzipCompressorInputStream(final InputStream inputStream, final boolean decompressConcatenated, final Charset fileNameCharset) throws IOException {
countingStream = BoundedInputStream.builder().setInputStream(inputStream).get();
// Mark support is strictly needed for concatenated files only,
// but it's simpler if it is always available.
- if (countingStream.markSupported()) {
- in = countingStream;
- } else {
- in = new BufferedInputStream(countingStream);
- }
-
+ in = countingStream.markSupported() ? countingStream : new BufferedInputStream(countingStream);
this.decompressConcatenated = decompressConcatenated;
+ this.fileNameCharset = fileNameCharset;
init(true);
}
@@ -161,17 +236,18 @@ public class GzipCompressorInputStream extends CompressorInputStream implements
*/
@Override
public void close() throws IOException {
- if (inf != null) {
- inf.end();
- inf = null;
+ if (inflater != null) {
+ inflater.end();
+ inflater = null;
}
-
if (this.in != System.in) {
this.in.close();
}
}
/**
+ * {@inheritDoc}.
+ *
* @since 1.17
*/
@Override
@@ -193,7 +269,7 @@ public class GzipCompressorInputStream extends CompressorInputStream implements
if (!isFirstMember && !decompressConcatenated) { // at least one must be true
throw new IllegalStateException("Unexpected: isFirstMember and decompressConcatenated are both false!");
}
-
+ parameters.setFileNameCharset(fileNameCharset);
// Check the magic bytes without a possibility of EOFException.
final int magic0 = in.read();
@@ -228,7 +304,7 @@ public class GzipCompressorInputStream extends CompressorInputStream implements
parameters.setCompressionLevel(Deflater.BEST_SPEED);
break;
default:
- // ignored for now
+ parameters.setCompressionLevel(Deflater.DEFAULT_COMPRESSION);
break;
}
parameters.setOperatingSystem(inData.readUnsignedByte());
@@ -244,12 +320,12 @@ public class GzipCompressorInputStream extends CompressorInputStream implements
// Original file name
if ((flg & GzipUtils.FNAME) != 0) {
- parameters.setFileName(new String(readToNull(inData), GzipUtils.GZIP_ENCODING));
+ parameters.setFileName(new String(readToNull(inData), parameters.getFileNameCharset()));
}
// Comment
if ((flg & GzipUtils.FCOMMENT) != 0) {
- parameters.setComment(new String(readToNull(inData), GzipUtils.GZIP_ENCODING));
+ parameters.setComment(new String(readToNull(inData), parameters.getFileNameCharset()));
}
// Header "CRC16" which is actually a truncated CRC32 (which isn't
@@ -263,7 +339,7 @@ public class GzipCompressorInputStream extends CompressorInputStream implements
}
// Reset
- inf.reset();
+ inflater.reset();
crc.reset();
return true;
@@ -291,7 +367,7 @@ public class GzipCompressorInputStream extends CompressorInputStream implements
int size = 0;
while (len > 0) {
- if (inf.needsInput()) {
+ if (inflater.needsInput()) {
// Remember the current position because we may need to
// rewind after reading too much input.
in.mark(buf.length);
@@ -301,14 +377,14 @@ public class GzipCompressorInputStream extends CompressorInputStream implements
throw new EOFException();
}
- inf.setInput(buf, 0, bufUsed);
+ inflater.setInput(buf, 0, bufUsed);
}
final int ret;
try {
- ret = inf.inflate(b, off, len);
+ ret = inflater.inflate(b, off, len);
} catch (final DataFormatException e) { // NOSONAR
- throw new IOException("Gzip-compressed data is corrupt");
+ throw new IOException("Gzip-compressed data is corrupt", e);
}
crc.update(b, off, ret);
@@ -317,12 +393,12 @@ public class GzipCompressorInputStream extends CompressorInputStream implements
size += ret;
count(ret);
- if (inf.finished()) {
+ if (inflater.finished()) {
// We may have read too many bytes. Rewind the read
// position to match the actual amount used.
in.reset();
- final int skipAmount = bufUsed - inf.getRemaining();
+ final int skipAmount = bufUsed - inflater.getRemaining();
if (org.apache.commons.io.IOUtils.skip(in, skipAmount) != skipAmount) {
throw new IOException();
}
@@ -341,14 +417,14 @@ public class GzipCompressorInputStream extends CompressorInputStream implements
// Uncompressed size modulo 2^32 (ISIZE in the spec)
final long isize = ByteUtils.fromLittleEndian(inData, 4);
- if (isize != (inf.getBytesWritten() & 0xffffffffL)) {
+ if (isize != (inflater.getBytesWritten() & 0xffffffffL)) {
throw new IOException("Gzip-compressed data is corrupt" + "(uncompressed size mismatch)");
}
// See if this is the end of the file.
if (!decompressConcatenated || !init(false)) {
- inf.end();
- inf = null;
+ inflater.end();
+ inflater = null;
endReached = true;
return size == 0 ? -1 : size;
}
diff --git a/src/test/java/org/apache/commons/compress/compressors/gzip/GzipCompressorOutputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/gzip/GzipCompressorOutputStreamTest.java
index 08eda6c04..f2779cd87 100644
--- a/src/test/java/org/apache/commons/compress/compressors/gzip/GzipCompressorOutputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/gzip/GzipCompressorOutputStreamTest.java
@@ -66,22 +66,32 @@ public class GzipCompressorOutputStreamTest {
Files.write(tempSourceFile, bytes);
final Path targetFile = Files.createTempFile(EXPECTED_BASE_NAME, ".gz");
final GzipParameters parameters = new GzipParameters();
- // If your system is Windows with Chinese, and your file name is Chinese, you need set the fileNameCharset to GBK
- // otherwise your file name is different with use GzipCompressorOutputStream without set GzipParameters
- // and the same situation in Linux, need set the fileNameCharset to UTF-8
+ // If your system is Windows with Chinese, and your file name is Chinese, you need set the fileNameCharset to "GBK"
+ // otherwise your file name is different using GzipCompressorOutputStream without a GzipParameters.
+ // On Linux, set the fileNameCharset to UTF-8.
parameters.setFileNameCharset(fileNameCharset);
assertEquals(fileNameCharset, parameters.getFileNameCharset());
parameters.setFileName(EXPECTED_FILE_NAME);
+ parameters.setComment("Comment on " + EXPECTED_FILE_NAME);
try (OutputStream fos = Files.newOutputStream(targetFile);
GzipCompressorOutputStream gos = new GzipCompressorOutputStream(fos, parameters)) {
gos.write(tempSourceFile);
}
+ // Old construction doesn't allow configuration of reading the file name and comment Charset.
try (GzipCompressorInputStream gis = new GzipCompressorInputStream(Files.newInputStream(targetFile))) {
final byte[] fileNameBytes = gis.getMetaData().getFileName().getBytes(StandardCharsets.ISO_8859_1);
final String unicodeFileName = new String(fileNameBytes, fileNameCharset);
assertEquals(expected, unicodeFileName);
assertArrayEquals(bytes, IOUtils.toByteArray(gis));
}
+ // Construction allows configuration of reading the file name and comment Charset.
+ try (GzipCompressorInputStream gis = GzipCompressorInputStream.builder().setPath(targetFile).setFileNameCharset(fileNameCharset).get()) {
+ final byte[] fileNameBytes = gis.getMetaData().getFileName().getBytes(fileNameCharset);
+ final String unicodeFileName = new String(fileNameBytes, fileNameCharset);
+ assertEquals(expected, unicodeFileName);
+ assertArrayEquals(bytes, IOUtils.toByteArray(gis));
+ assertEquals(parameters, gis.getMetaData());
+ }
}
/**
@@ -160,6 +170,7 @@ public class GzipCompressorOutputStreamTest {
}
try (GzipCompressorInputStream gis = new GzipCompressorInputStream(Files.newInputStream(targetFile))) {
final ExtraField extra2 = gis.getMetaData().getExtraField();
+ assertEquals(parameters, gis.getMetaData());
assertEquals(subFieldCount == 0, extra2.isEmpty());
assertEquals(subFieldCount, extra2.size());
assertEquals(4 * subFieldCount + subFieldCount * payloadSize, extra2.getEncodedSize());
@@ -183,9 +194,7 @@ public class GzipCompressorOutputStreamTest {
final ExtraField extra = new ExtraField();
assertEquals(0, extra.toByteArray().length);
assertFalse(extra.iterator().hasNext());
- extra.forEach(e -> {
- fail("Not emprt");
- });
+ extra.forEach(e -> fail("Not empty."));
assertThrows(IndexOutOfBoundsException.class, () -> extra.getSubField(0));
}
@@ -279,6 +288,7 @@ public class GzipCompressorOutputStreamTest {
assertEquals("AAAA", metaData.getFileName());
assertEquals("ZZZZ", metaData.getComment());
assertEquals(OS.UNIX, metaData.getOS());
+ assertEquals(parameters, metaData);
}
// verify that the constructor normally fails on bad HCRC
assertThrows(ZipException.class, () -> {