You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/05/09 07:27:58 UTC
[1/2] commons-compress git commit: COMPRESS-391: Allow alignment on
zip content
Repository: commons-compress
Updated Branches:
refs/heads/COMPRESS-391 [created] 620196621
COMPRESS-391: Allow alignment on zip content
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/9426aaaa
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/9426aaaa
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/9426aaaa
Branch: refs/heads/COMPRESS-391
Commit: 9426aaaaa51c84f1a00fc6052bd8951e792ca957
Parents: 4ac67bf
Author: Zbynek Vyskovsky <kv...@gmail.com>
Authored: Thu May 4 20:44:44 2017 -0700
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Tue May 9 09:05:01 2017 +0200
----------------------------------------------------------------------
.../compress/archivers/zip/ZipArchiveEntry.java | 24 +++++++
.../archivers/zip/ZipArchiveOutputStream.java | 39 +++++++++--
.../compress/archivers/zip/ZipFileTest.java | 71 ++++++++++++++++++++
3 files changed, 130 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/9426aaaa/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java
index f769cf5..b626ea6 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java
@@ -85,6 +85,7 @@ public class ZipArchiveEntry extends java.util.zip.ZipEntry
private int platform = PLATFORM_FAT;
private int rawFlag;
private long externalAttributes = 0;
+ private int alignment = 0;
private ZipExtraField[] extraFields;
private UnparseableExtraFieldData unparseableExtra = null;
private String name = null;
@@ -323,6 +324,29 @@ public class ZipArchiveEntry extends java.util.zip.ZipEntry
}
/**
+ * Gets currently configured alignment.
+ *
+ * @return
+ * alignment for this entry.
+ */
+ protected int getAlignment() {
+ return this.alignment;
+ }
+
+ /**
+ * Sets alignment for this entry.
+ *
+ * @param alignment
+ * requested alignment, 0 for default.
+ */
+ public void setAlignment(int alignment) {
+ if ((alignment & (alignment - 1)) != 0) {
+ throw new IllegalArgumentException("Invalid value for alignment, must be power of two: " + alignment);
+ }
+ this.alignment = alignment;
+ }
+
+ /**
* Replaces all currently attached extra fields with the new array.
* @param fields an array of extra fields
*/
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/9426aaaa/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
index e3066b0..72e3d5b 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
@@ -142,6 +142,17 @@ public class ZipArchiveOutputStream extends ArchiveOutputStream {
@Deprecated
public static final int EFS_FLAG = GeneralPurposeBit.UFT8_NAMES_FLAG;
+ /**
+ * Size of the extra field header (id + length).
+ */
+ public static final int EXTRAFIELD_HEADER_SIZE = 4;
+
+ /**
+ * Extra field id used for padding (there is no special value documented,
+ * therefore USHORT_MAX seems to be good choice).
+ */
+ public static final int EXTRAFIELD_PADDING_ID = 0xffff;
+
private static final byte[] EMPTY = new byte[0];
/**
@@ -1026,8 +1037,8 @@ public class ZipArchiveOutputStream extends ArchiveOutputStream {
addUnicodeExtraFields(ze, encodable, name);
}
- final byte[] localHeader = createLocalFileHeader(ze, name, encodable, phased);
final long localHeaderStart = streamCompressor.getTotalBytesWritten();
+ final byte[] localHeader = createLocalFileHeader(ze, name, encodable, phased, localHeaderStart);
offsets.put(ze, localHeaderStart);
entry.localDataStart = localHeaderStart + LFH_CRC_OFFSET; // At crc offset
writeCounted(localHeader);
@@ -1036,10 +1047,16 @@ public class ZipArchiveOutputStream extends ArchiveOutputStream {
private byte[] createLocalFileHeader(final ZipArchiveEntry ze, final ByteBuffer name, final boolean encodable,
- final boolean phased) {
+ final boolean phased, long archiveOffset) throws IOException {
final byte[] extra = ze.getLocalFileDataExtra();
final int nameLen = name.limit() - name.position();
- final int len= LFH_FILENAME_OFFSET + nameLen + extra.length;
+ int len= LFH_FILENAME_OFFSET + nameLen + extra.length;
+ int padding = 0;
+ int alignment = ze.getAlignment();
+ if (alignment > 1 && ((archiveOffset + len) & (alignment - 1)) != 0) {
+ padding = (int) ((-archiveOffset - len - EXTRAFIELD_HEADER_SIZE) & (alignment - 1));
+ len += EXTRAFIELD_HEADER_SIZE+padding;
+ }
final byte[] buf = new byte[len];
System.arraycopy(LFH_SIG, 0, buf, LFH_SIG_OFFSET, WORD);
@@ -1091,13 +1108,27 @@ public class ZipArchiveOutputStream extends ArchiveOutputStream {
// file name length
putShort(nameLen, buf, LFH_FILENAME_LENGTH_OFFSET);
+ int totalExtra = extra.length + (padding > 0 ? padding + EXTRAFIELD_HEADER_SIZE : 0);
+ if (totalExtra > 0xffff) {
+ throw new IOException("Too much data for extra fields and padding"+
+ ", extra="+extra.length+
+ ", padding="+padding);
+ }
// extra field length
- putShort(extra.length, buf, LFH_EXTRA_LENGTH_OFFSET);
+ putShort(totalExtra, buf, LFH_EXTRA_LENGTH_OFFSET);
// file name
System.arraycopy( name.array(), name.arrayOffset(), buf, LFH_FILENAME_OFFSET, nameLen);
+ // extra fields
System.arraycopy(extra, 0, buf, LFH_FILENAME_OFFSET + nameLen, extra.length);
+
+ // padding
+ if (padding > 0) {
+ putShort(EXTRAFIELD_PADDING_ID, buf, LFH_FILENAME_OFFSET + nameLen + extra.length);
+ putShort(padding, buf, LFH_FILENAME_OFFSET + nameLen + extra.length + 2);
+ }
+
return buf;
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/9426aaaa/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java b/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
index 8455bd7..5b7b900 100644
--- a/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
+++ b/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
@@ -28,6 +28,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -460,6 +461,76 @@ public class ZipFileTest {
}
}
+ /**
+ * Test entries alignment.
+ */
+ @Test
+ public void testEntryAlignment() throws Exception {
+ SeekableInMemoryByteChannel zipContent = new SeekableInMemoryByteChannel();
+ try (ZipArchiveOutputStream zipOutput = new ZipArchiveOutputStream(zipContent)) {
+ ZipArchiveEntry inflatedEntry = new ZipArchiveEntry("inflated.txt");
+ inflatedEntry.setMethod(ZipEntry.DEFLATED);
+ inflatedEntry.setAlignment(1024);
+ zipOutput.putArchiveEntry(inflatedEntry);
+ zipOutput.write("Hello Deflated\n".getBytes(Charset.forName("UTF-8")));
+ zipOutput.closeArchiveEntry();
+
+ ZipArchiveEntry storedEntry = new ZipArchiveEntry("stored.txt");
+ storedEntry.setMethod(ZipEntry.STORED);
+ storedEntry.setAlignment(1024);
+ zipOutput.putArchiveEntry(storedEntry);
+ zipOutput.write("Hello Stored\n".getBytes(Charset.forName("UTF-8")));
+ zipOutput.closeArchiveEntry();
+
+ }
+
+ try (ZipFile zf = new ZipFile(new SeekableInMemoryByteChannel(
+ Arrays.copyOfRange(zipContent.array(), 0, (int)zipContent.size())
+ ))) {
+ ZipArchiveEntry inflatedEntry = zf.getEntry("inflated.txt");
+ assertNotEquals(-1L, inflatedEntry.getCompressedSize());
+ assertNotEquals(-1L, inflatedEntry.getSize());
+ assertEquals(0L, inflatedEntry.getDataOffset()%1024);
+ try (InputStream stream = zf.getInputStream(inflatedEntry)) {
+ Assert.assertEquals("Hello Deflated\n",
+ new String(IOUtils.toByteArray(stream), Charset.forName("UTF-8")));
+ }
+ ZipArchiveEntry storedEntry = zf.getEntry("stored.txt");
+ assertNotEquals(-1L, storedEntry.getCompressedSize());
+ assertNotEquals(-1L, storedEntry.getSize());
+ assertEquals(0L, inflatedEntry.getDataOffset()%1024);
+ try (InputStream stream = zf.getInputStream(storedEntry)) {
+ Assert.assertEquals("Hello Stored\n",
+ new String(IOUtils.toByteArray(stream), Charset.forName("UTF-8")));
+ }
+ }
+ }
+
+ /**
+ * Test too big alignment, resulting into exceeding extra field limit.
+ */
+ @Test(expected = IOException.class)
+ public void testEntryAlignmentExceed() throws Exception {
+ SeekableInMemoryByteChannel zipContent = new SeekableInMemoryByteChannel();
+ try (ZipArchiveOutputStream zipOutput = new ZipArchiveOutputStream(zipContent)) {
+ ZipArchiveEntry inflatedEntry = new ZipArchiveEntry("inflated.txt");
+ inflatedEntry.setMethod(ZipEntry.STORED);
+ inflatedEntry.setAlignment(0x20000);
+ zipOutput.putArchiveEntry(inflatedEntry);
+ zipOutput.write("Hello Stored\n".getBytes(Charset.forName("UTF-8")));
+ zipOutput.closeArchiveEntry();
+ }
+ }
+
+ /**
+ * Test non power of 2 alignment.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidAlignment() throws Exception {
+ ZipArchiveEntry entry = new ZipArchiveEntry("dummy");
+ entry.setAlignment(3);
+ }
+
private void assertAllReadMethods(byte[] expected, ZipFile zipFile, ZipArchiveEntry entry) {
// simple IOUtil read
try (InputStream stream = zf.getInputStream(entry)) {
[2/2] commons-compress git commit: COMPRESS-391 implement padding as
ZipExtraField class
Posted by bo...@apache.org.
COMPRESS-391 implement padding as ZipExtraField class
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/62019662
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/62019662
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/62019662
Branch: refs/heads/COMPRESS-391
Commit: 620196621e15a87cdd5e3382504bd8a9829f4698
Parents: 9426aaa
Author: Stefan Bodewig <bo...@apache.org>
Authored: Tue May 9 09:27:16 2017 +0200
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Tue May 9 09:27:16 2017 +0200
----------------------------------------------------------------------
.../compress/archivers/zip/ExtraFieldUtils.java | 1 +
.../archivers/zip/PaddingExtraField.java | 82 ++++++++++++++++++++
.../compress/archivers/zip/ZipArchiveEntry.java | 5 +-
.../archivers/zip/ZipArchiveOutputStream.java | 31 ++------
.../compress/archivers/zip/ZipShort.java | 2 +
.../compress/archivers/zip/ZipFileTest.java | 5 +-
6 files changed, 96 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/62019662/src/main/java/org/apache/commons/compress/archivers/zip/ExtraFieldUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ExtraFieldUtils.java b/src/main/java/org/apache/commons/compress/archivers/zip/ExtraFieldUtils.java
index a21c858..c510efa 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ExtraFieldUtils.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ExtraFieldUtils.java
@@ -52,6 +52,7 @@ public class ExtraFieldUtils {
register(X0016_CertificateIdForCentralDirectory.class);
register(X0017_StrongEncryptionHeader.class);
register(X0019_EncryptionRecipientCertificateList.class);
+ register(PaddingExtraField.class);
}
/**
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/62019662/src/main/java/org/apache/commons/compress/archivers/zip/PaddingExtraField.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/PaddingExtraField.java b/src/main/java/org/apache/commons/compress/archivers/zip/PaddingExtraField.java
new file mode 100644
index 0000000..28df2e6
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/PaddingExtraField.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress.archivers.zip;
+
+import java.util.zip.ZipException;
+
+/**
+ * An extra field who's sole purpose is to pad the local file header
+ * so that the entry's data starts at a certain position.
+ *
+ * <p>The actual content of the padding is ignored and not retained
+ * when reading a padding field.</p>
+ *
+ * @since 1.14
+ */
+public class PaddingExtraField implements ZipExtraField {
+
+ /**
+ * Extra field id used for padding (there is no special value documented,
+ * therefore USHORT_MAX seems to be good choice).
+ */
+ private static final ZipShort ID = new ZipShort(0xffff);
+
+ private int len = 0;
+
+ public PaddingExtraField() {
+ }
+
+ public PaddingExtraField(int len) {
+ this.len = len;
+ }
+
+ @Override
+ public ZipShort getHeaderId() {
+ return ID;
+ }
+
+ @Override
+ public ZipShort getLocalFileDataLength() {
+ return new ZipShort(len);
+ }
+
+ @Override
+ public ZipShort getCentralDirectoryLength() {
+ return ZipShort.ZERO;
+ }
+
+ @Override
+ public byte[] getLocalFileDataData() {
+ return new byte[len];
+ }
+
+ @Override
+ public byte[] getCentralDirectoryData() {
+ return new byte[0];
+ }
+
+ @Override
+ public void parseFromLocalFileData(byte[] buffer, int offset, int length) {
+ len = length;
+ }
+
+ @Override
+ public void parseFromCentralDirectoryData(byte[] buffer, int offset, int length) {
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/62019662/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java
index b626ea6..ffcb953 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java
@@ -340,8 +340,9 @@ public class ZipArchiveEntry extends java.util.zip.ZipEntry
* requested alignment, 0 for default.
*/
public void setAlignment(int alignment) {
- if ((alignment & (alignment - 1)) != 0) {
- throw new IllegalArgumentException("Invalid value for alignment, must be power of two: " + alignment);
+ if ((alignment & (alignment - 1)) != 0 || alignment > 0xffff) {
+ throw new IllegalArgumentException("Invalid value for alignment, must be power of two and no bigger than "
+ + 0xffff + " but is " + alignment);
}
this.alignment = alignment;
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/62019662/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
index 72e3d5b..7161301 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
@@ -143,16 +143,10 @@ public class ZipArchiveOutputStream extends ArchiveOutputStream {
public static final int EFS_FLAG = GeneralPurposeBit.UFT8_NAMES_FLAG;
/**
- * Size of the extra field header (id + length).
+ * Size of an extra field field header (id + length).
*/
public static final int EXTRAFIELD_HEADER_SIZE = 4;
- /**
- * Extra field id used for padding (there is no special value documented,
- * therefore USHORT_MAX seems to be good choice).
- */
- public static final int EXTRAFIELD_PADDING_ID = 0xffff;
-
private static final byte[] EMPTY = new byte[0];
/**
@@ -1048,14 +1042,15 @@ public class ZipArchiveOutputStream extends ArchiveOutputStream {
private byte[] createLocalFileHeader(final ZipArchiveEntry ze, final ByteBuffer name, final boolean encodable,
final boolean phased, long archiveOffset) throws IOException {
- final byte[] extra = ze.getLocalFileDataExtra();
+ byte[] extra = ze.getLocalFileDataExtra();
final int nameLen = name.limit() - name.position();
int len= LFH_FILENAME_OFFSET + nameLen + extra.length;
- int padding = 0;
int alignment = ze.getAlignment();
if (alignment > 1 && ((archiveOffset + len) & (alignment - 1)) != 0) {
- padding = (int) ((-archiveOffset - len - EXTRAFIELD_HEADER_SIZE) & (alignment - 1));
- len += EXTRAFIELD_HEADER_SIZE+padding;
+ int padding = (int) ((-archiveOffset - len - EXTRAFIELD_HEADER_SIZE) & (alignment - 1));
+ ze.addExtraField(new PaddingExtraField(padding));
+ extra = ze.getLocalFileDataExtra();
+ len += EXTRAFIELD_HEADER_SIZE + padding;
}
final byte[] buf = new byte[len];
@@ -1108,14 +1103,8 @@ public class ZipArchiveOutputStream extends ArchiveOutputStream {
// file name length
putShort(nameLen, buf, LFH_FILENAME_LENGTH_OFFSET);
- int totalExtra = extra.length + (padding > 0 ? padding + EXTRAFIELD_HEADER_SIZE : 0);
- if (totalExtra > 0xffff) {
- throw new IOException("Too much data for extra fields and padding"+
- ", extra="+extra.length+
- ", padding="+padding);
- }
// extra field length
- putShort(totalExtra, buf, LFH_EXTRA_LENGTH_OFFSET);
+ putShort(extra.length, buf, LFH_EXTRA_LENGTH_OFFSET);
// file name
System.arraycopy( name.array(), name.arrayOffset(), buf, LFH_FILENAME_OFFSET, nameLen);
@@ -1123,12 +1112,6 @@ public class ZipArchiveOutputStream extends ArchiveOutputStream {
// extra fields
System.arraycopy(extra, 0, buf, LFH_FILENAME_OFFSET + nameLen, extra.length);
- // padding
- if (padding > 0) {
- putShort(EXTRAFIELD_PADDING_ID, buf, LFH_FILENAME_OFFSET + nameLen + extra.length);
- putShort(padding, buf, LFH_FILENAME_OFFSET + nameLen + extra.length + 2);
- }
-
return buf;
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/62019662/src/main/java/org/apache/commons/compress/archivers/zip/ZipShort.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipShort.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipShort.java
index 7d182f8..65acf21 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipShort.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipShort.java
@@ -27,6 +27,8 @@ import org.apache.commons.compress.utils.ByteUtils;
* @Immutable
*/
public final class ZipShort implements Cloneable, Serializable {
+ public static final ZipShort ZERO = new ZipShort(0);
+
private static final long serialVersionUID = 1L;
private final int value;
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/62019662/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java b/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
index 5b7b900..0c94368 100644
--- a/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
+++ b/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
@@ -509,16 +509,13 @@ public class ZipFileTest {
/**
* Test too big alignment, resulting into exceeding extra field limit.
*/
- @Test(expected = IOException.class)
+ @Test(expected = IllegalArgumentException.class)
public void testEntryAlignmentExceed() throws Exception {
SeekableInMemoryByteChannel zipContent = new SeekableInMemoryByteChannel();
try (ZipArchiveOutputStream zipOutput = new ZipArchiveOutputStream(zipContent)) {
ZipArchiveEntry inflatedEntry = new ZipArchiveEntry("inflated.txt");
inflatedEntry.setMethod(ZipEntry.STORED);
inflatedEntry.setAlignment(0x20000);
- zipOutput.putArchiveEntry(inflatedEntry);
- zipOutput.write("Hello Stored\n".getBytes(Charset.forName("UTF-8")));
- zipOutput.closeArchiveEntry();
}
}