You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2018/04/22 18:10:16 UTC
[1/5] commons-compress git commit: COMPRESS-445 refactor statistics
test
Repository: commons-compress
Updated Branches:
refs/heads/master ac7f09c8c -> 36a5631eb
COMPRESS-445 refactor statistics test
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/d6b07a11
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/d6b07a11
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/d6b07a11
Branch: refs/heads/master
Commit: d6b07a11ab9e2c4eacecfb8dd1b66741817674be
Parents: ac7f09c
Author: Stefan Bodewig <bo...@apache.org>
Authored: Sun Apr 22 18:56:58 2018 +0200
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Sun Apr 22 18:56:58 2018 +0200
----------------------------------------------------------------------
.../commons/compress/archivers/ZipTestCase.java | 32 ++++++++++++++------
1 file changed, 22 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/d6b07a11/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java b/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java
index 4451a07..edba96a 100644
--- a/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java
+++ b/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java
@@ -604,16 +604,25 @@ public final class ZipTestCase extends AbstractTestCase {
}
@Test
- public void testInputStreamStatistics() throws IOException, ArchiveException {
- final File input = getFile("zipbomb.xlsx");
+ public void inputStreamStatisticsOfZipBombExcel() throws IOException, ArchiveException {
+ Map<String, List<Long>> expected = new HashMap<String, List<Long>>() {{
+ put("[Content_Types].xml", Arrays.asList(8390036L, 8600L));
+ put("xl/worksheets/sheet1.xml", Arrays.asList(1348L, 508L));
+ }};
+ testInputStreamStatistics("zipbomb.xlsx", expected);
+ }
+
+ private void testInputStreamStatistics(String fileName, Map<String, List<Long>> expectedStatistics)
+ throws IOException, ArchiveException {
+ final File input = getFile(fileName);
- final Map<String,List<List<Long>>> map = new HashMap<>();
+ final Map<String,List<List<Long>>> actualStatistics = new HashMap<>();
// stream access
try (final FileInputStream fis = new FileInputStream(input);
final ArchiveInputStream in = new ArchiveStreamFactory().createArchiveInputStream("zip", fis)) {
for (ArchiveEntry entry; (entry = in.getNextEntry()) != null; ) {
- readStream(in, entry, map);
+ readStream(in, entry, actualStatistics);
}
}
@@ -623,23 +632,26 @@ public final class ZipTestCase extends AbstractTestCase {
while (entries.hasMoreElements()) {
final ZipArchiveEntry zae = entries.nextElement();
try (InputStream in = zf.getInputStream(zae)) {
- readStream(in, zae, map);
+ readStream(in, zae, actualStatistics);
}
}
}
- assertEquals(Arrays.asList(8390036L, 8600L), map.get("[Content_Types].xml").get(0));
- assertEquals(Arrays.asList(1348L, 508L), map.get("xl/worksheets/sheet1.xml").get(0));
+ for (Map.Entry<String, List<Long>> me : expectedStatistics.entrySet()) {
+ assertEquals("Mismatch of stats with expected value for: " + me.getKey(),
+ me.getValue(), actualStatistics.get(me.getKey()).get(0));
+ }
// compare statistics of stream / file access
- for (Map.Entry<String,List<List<Long>>> me : map.entrySet()) {
- assertEquals("Mismatch of stats for: "+me.getKey(), me.getValue().get(0), me.getValue().get(1));
+ for (Map.Entry<String,List<List<Long>>> me : actualStatistics.entrySet()) {
+ assertEquals("Mismatch of stats for: " + me.getKey(),
+ me.getValue().get(0), me.getValue().get(1));
}
}
private void readStream(final InputStream in, final ArchiveEntry entry, final Map<String,List<List<Long>>> map) throws IOException {
final byte[] buf = new byte[4096];
- final InputStreamStatistics stats = (InputStreamStatistics)in;
+ final InputStreamStatistics stats = (InputStreamStatistics) in;
while (in.read(buf) != -1);
final String name = entry.getName();
[5/5] commons-compress git commit: COMPRESS-445 test cases for the
remaining ZIP methods
Posted by bo...@apache.org.
COMPRESS-445 test cases for the remaining ZIP methods
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/36a5631e
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/36a5631e
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/36a5631e
Branch: refs/heads/master
Commit: 36a5631ebe2cdb53d41428d1f17ce72ed618045c
Parents: 0db4e9a
Author: Stefan Bodewig <bo...@apache.org>
Authored: Sun Apr 22 20:09:34 2018 +0200
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Sun Apr 22 20:09:34 2018 +0200
----------------------------------------------------------------------
.../commons/compress/archivers/ZipTestCase.java | 53 ++++++++++++++++++--
1 file changed, 48 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/36a5631e/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java b/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java
index edba96a..fc98824 100644
--- a/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java
+++ b/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java
@@ -612,6 +612,49 @@ public final class ZipTestCase extends AbstractTestCase {
testInputStreamStatistics("zipbomb.xlsx", expected);
}
+ @Test
+ @org.junit.Ignore("ZipFile and ZipArchiveInputStream agree but are different from unzip -l -v")
+ public void inputStreamStatisticsForImplodedEntry() throws IOException, ArchiveException {
+ Map<String, List<Long>> expected = new HashMap<String, List<Long>>() {{
+ put("LICENSE.TXT", Arrays.asList(11560L, 4131L));
+ }};
+ testInputStreamStatistics("imploding-8Kdict-3trees.zip", expected);
+ }
+
+ @Test
+ public void inputStreamStatisticsForShrunkEntry() throws IOException, ArchiveException {
+ Map<String, List<Long>> expected = new HashMap<String, List<Long>>() {{
+ put("TEST1.XML", Arrays.asList(76L, 66L));
+ put("TEST2.XML", Arrays.asList(81L, 76L));
+ }};
+ testInputStreamStatistics("SHRUNK.ZIP", expected);
+ }
+
+ @Test
+ @org.junit.Ignore("result from ZipArchiveInputStream is wrong")
+ public void inputStreamStatisticsForStoredEntry() throws IOException, ArchiveException {
+ Map<String, List<Long>> expected = new HashMap<String, List<Long>>() {{
+ put("test.txt", Arrays.asList(5L, 5L));
+ }};
+ testInputStreamStatistics("COMPRESS-264.zip", expected);
+ }
+
+ @Test
+ public void inputStreamStatisticsForBzip2Entry() throws IOException, ArchiveException {
+ Map<String, List<Long>> expected = new HashMap<String, List<Long>>() {{
+ put("lots-of-as", Arrays.asList(42L, 39L));
+ }};
+ testInputStreamStatistics("bzip2-zip.zip", expected);
+ }
+
+ @Test
+ public void inputStreamStatisticsForDeflate64Entry() throws IOException, ArchiveException {
+ Map<String, List<Long>> expected = new HashMap<String, List<Long>>() {{
+ put("input2", Arrays.asList(3072L, 2111L));
+ }};
+ testInputStreamStatistics("COMPRESS-380/COMPRESS-380.zip", expected);
+ }
+
private void testInputStreamStatistics(String fileName, Map<String, List<Long>> expectedStatistics)
throws IOException, ArchiveException {
final File input = getFile(fileName);
@@ -637,16 +680,16 @@ public final class ZipTestCase extends AbstractTestCase {
}
}
- for (Map.Entry<String, List<Long>> me : expectedStatistics.entrySet()) {
- assertEquals("Mismatch of stats with expected value for: " + me.getKey(),
- me.getValue(), actualStatistics.get(me.getKey()).get(0));
- }
-
// compare statistics of stream / file access
for (Map.Entry<String,List<List<Long>>> me : actualStatistics.entrySet()) {
assertEquals("Mismatch of stats for: " + me.getKey(),
me.getValue().get(0), me.getValue().get(1));
}
+
+ for (Map.Entry<String, List<Long>> me : expectedStatistics.entrySet()) {
+ assertEquals("Mismatch of stats with expected value for: " + me.getKey(),
+ me.getValue(), actualStatistics.get(me.getKey()).get(0));
+ }
}
private void readStream(final InputStream in, final ArchiveEntry entry, final Map<String,List<List<Long>>> map) throws IOException {
[3/5] commons-compress git commit: COMPRESS-445 make ZipFile return
statistics for stored entries
Posted by bo...@apache.org.
COMPRESS-445 make ZipFile return statistics for stored entries
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/74abd5b7
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/74abd5b7
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/74abd5b7
Branch: refs/heads/master
Commit: 74abd5b73c28bb625d0c74323e64a941c4dc02e7
Parents: b8f7d77
Author: Stefan Bodewig <bo...@apache.org>
Authored: Sun Apr 22 20:08:02 2018 +0200
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Sun Apr 22 20:08:02 2018 +0200
----------------------------------------------------------------------
.../commons/compress/archivers/zip/ZipFile.java | 20 +++++++++++++++++++-
1 file changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/74abd5b7/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
index 5aa8165..868f581 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
@@ -44,7 +44,9 @@ import java.util.zip.ZipException;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.deflate64.Deflate64CompressorInputStream;
+import org.apache.commons.compress.utils.CountingInputStream;
import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.compress.utils.InputStreamStatistics;
import static org.apache.commons.compress.archivers.zip.ZipConstants.DWORD;
import static org.apache.commons.compress.archivers.zip.ZipConstants.SHORT;
@@ -486,7 +488,7 @@ public class ZipFile implements Closeable {
new BufferedInputStream(createBoundedInputStream(start, ze.getCompressedSize())); //NOSONAR
switch (ZipMethod.getMethodByCode(ze.getMethod())) {
case STORED:
- return is;
+ return new StoredStatisticsStream(is);
case UNSHRINKING:
return new UnshrinkingInputStream(is);
case IMPLODING:
@@ -1256,4 +1258,20 @@ public class ZipFile implements Closeable {
return false;
}
}
+
+ private static class StoredStatisticsStream extends CountingInputStream implements InputStreamStatistics {
+ StoredStatisticsStream(InputStream in) {
+ super(in);
+ }
+
+ @Override
+ public long getCompressedCount() {
+ return super.getBytesRead();
+ }
+
+ @Override
+ public long getUncompressedCount() {
+ return getCompressedCount();
+ }
+ }
}
[4/5] commons-compress git commit: COMPRESS-445 document ZipFile's
new contract
Posted by bo...@apache.org.
COMPRESS-445 document ZipFile's new contract
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/0db4e9a6
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/0db4e9a6
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/0db4e9a6
Branch: refs/heads/master
Commit: 0db4e9a63319f9fab2d347181533c127977f59fb
Parents: 74abd5b
Author: Stefan Bodewig <bo...@apache.org>
Authored: Sun Apr 22 20:09:16 2018 +0200
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Sun Apr 22 20:09:16 2018 +0200
----------------------------------------------------------------------
.../java/org/apache/commons/compress/archivers/zip/ZipFile.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/0db4e9a6/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
index 868f581..5e7cfa0 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
@@ -469,7 +469,8 @@ public class ZipFile implements Closeable {
* Returns an InputStream for reading the contents of the given entry.
*
* @param ze the entry to get the stream for.
- * @return a stream to read the entry from.
+ * @return a stream to read the entry from. The returned stream
+ * implements {@link InputStreamStatistics}.
* @throws IOException if unable to create an input stream from the zipentry
*/
public InputStream getInputStream(final ZipArchiveEntry ze)
[2/5] commons-compress git commit: COMPRESS-445 can't use decoder
after stream has been exhausted
Posted by bo...@apache.org.
COMPRESS-445 can't use decoder after stream has been exhausted
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/b8f7d772
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/b8f7d772
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/b8f7d772
Branch: refs/heads/master
Commit: b8f7d772c96b0d1fa3b7a82c741d336d052940b2
Parents: d6b07a1
Author: Stefan Bodewig <bo...@apache.org>
Authored: Sun Apr 22 19:54:50 2018 +0200
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Sun Apr 22 19:54:50 2018 +0200
----------------------------------------------------------------------
.../compressors/deflate64/Deflate64CompressorInputStream.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b8f7d772/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
index 36b8a83..a0efd35 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
@@ -34,6 +34,7 @@ import static org.apache.commons.compress.utils.IOUtils.closeQuietly;
public class Deflate64CompressorInputStream extends CompressorInputStream implements InputStreamStatistics {
private InputStream originalStream;
private HuffmanDecoder decoder;
+ private long compressedBytesRead;
private final byte[] oneByte = new byte[1];
/**
@@ -78,6 +79,7 @@ public class Deflate64CompressorInputStream extends CompressorInputStream implem
int read = -1;
if (decoder != null) {
read = decoder.decode(b, off, len);
+ compressedBytesRead = decoder.getBytesRead();
count(read);
if (read == -1) {
closeDecoder();
@@ -105,7 +107,7 @@ public class Deflate64CompressorInputStream extends CompressorInputStream implem
*/
@Override
public long getCompressedCount() {
- return decoder.getBytesRead();
+ return compressedBytesRead;
}
/**