You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/04/06 13:18:30 UTC
[1/3] commons-compress git commit: cosmetics
Repository: commons-compress
Updated Branches:
refs/heads/master 3381e41ab -> 5ce60be9d
cosmetics
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/47213feb
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/47213feb
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/47213feb
Branch: refs/heads/master
Commit: 47213feb954bef78d646da4f4ffe6a8156c7d3f5
Parents: 3381e41
Author: Stefan Bodewig <bo...@apache.org>
Authored: Thu Apr 6 10:32:33 2017 +0200
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Thu Apr 6 10:32:33 2017 +0200
----------------------------------------------------------------------
.../lz4/BlockLZ4CompressorOutputStream.java | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/47213feb/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
index d3249db..dfce36e 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
@@ -385,14 +385,12 @@ public class BlockLZ4CompressorOutputStream extends CompressorOutputStream {
}
Pair splitCandidate = lastPairs.get(0);
int stillNeeded = MIN_OFFSET_OF_LAST_BACK_REFERENCE - toExpand;
- if (splitCandidate.hasBackReference()
- && splitCandidate.backReferenceLength() >= MIN_BACK_REFERENCE_LENGTH + stillNeeded) {
+ int brLen = splitCandidate.hasBackReference() ? splitCandidate.backReferenceLength() : 0;
+ if (splitCandidate.hasBackReference() && brLen >= MIN_BACK_REFERENCE_LENGTH + stillNeeded) {
replacement.prependLiteral(expand(toExpand + stillNeeded, stillNeeded));
- pairs.add(splitCandidate.splitWithNewBackReferenceLengthOf(splitCandidate.backReferenceLength()
- - stillNeeded));
+ pairs.add(splitCandidate.splitWithNewBackReferenceLengthOf(brLen - stillNeeded));
} else {
if (splitCandidate.hasBackReference()) {
- int brLen = splitCandidate.backReferenceLength();
replacement.prependLiteral(expand(toExpand + brLen, brLen));
}
splitCandidate.prependTo(replacement);
@@ -487,9 +485,9 @@ public class BlockLZ4CompressorOutputStream extends CompressorOutputStream {
return brLength;
}
private void prependTo(Pair other) {
- Iterator<byte[]> litsBackwards = literals.descendingIterator();
- while (litsBackwards.hasNext()) {
- other.prependLiteral(litsBackwards.next());
+ Iterator<byte[]> listBackwards = literals.descendingIterator();
+ while (listBackwards.hasNext()) {
+ other.prependLiteral(listBackwards.next());
}
}
private Pair splitWithNewBackReferenceLengthOf(int newBackReferenceLength) {
[3/3] commons-compress git commit: fix calculation of self-copy
boundaries when expanding back-references
Posted by bo...@apache.org.
fix calculation of self-copy boundaries when expanding back-references
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/5ce60be9
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/5ce60be9
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/5ce60be9
Branch: refs/heads/master
Commit: 5ce60be9d55c62b72262680249ad933850aaa064
Parents: 62202d2
Author: Stefan Bodewig <bo...@apache.org>
Authored: Thu Apr 6 15:18:00 2017 +0200
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Thu Apr 6 15:18:00 2017 +0200
----------------------------------------------------------------------
.../lz4/BlockLZ4CompressorOutputStream.java | 4 ++--
.../lz4/BlockLZ4CompressorOutputStreamTest.java | 23 ++++++++++++++++++++
2 files changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/5ce60be9/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
index dfce36e..71ca713 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
@@ -264,9 +264,9 @@ public class BlockLZ4CompressorOutputStream extends CompressorOutputStream {
copyOffset = blockOffset + block.length - offsetRemaining;
copyLen = Math.min(lengthRemaining, block.length - copyOffset);
} else {
- // offsetRemaining is negative and points into the expanded bytes
+ // offsetRemaining is negative or 0 and points into the expanded bytes
block = expanded;
- copyOffset = writeOffset + offsetRemaining;
+ copyOffset = -offsetRemaining;
copyLen = Math.min(lengthRemaining, writeOffset + offsetRemaining);
}
System.arraycopy(block, copyOffset, expanded, writeOffset, copyLen);
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/5ce60be9/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStreamTest.java
index 45bf684..fca3f88 100644
--- a/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStreamTest.java
@@ -319,6 +319,29 @@ public class BlockLZ4CompressorOutputStreamTest {
Assert.assertArrayEquals(expected, compressed);
}
+ @Test
+ public void rewritingWithFinalBackreferenceAndOffsetBiggerThan1() throws IOException {
+ // this caused trouble when expandFromList() fell into the "offsetRemaining is negative" self-copy case as the
+ // calculation of copyOffset was wrong
+ byte[] toCompress = prepareExpected(25);
+ for (int i = 0; i < toCompress.length; i += 4) {
+ toCompress[i] = 1;
+ }
+ // LZ77Compressor creates a four byte literal and a back-reference with offset 4 and length 21
+ // we'll need to split the back-reference and chop off the last 12 bytes
+ byte[] compressed = compress(toCompress);
+ byte[] expected = prepareExpected(1 + 4 + 2 + 1 + 12);
+ expected[0] = (byte) ((4<<4) | 5);
+ expected[1] = 1;
+ expected[5] = 4;
+ expected[6] = 0;
+ expected[7] = (byte) (12<<4);
+ for (int i = 11; i < expected.length; i += 4) {
+ expected[i] = 1;
+ }
+ Assert.assertArrayEquals(expected, compressed);
+ }
+
private byte[] compress(int length) throws IOException {
return compress(length, 0);
}
[2/3] commons-compress git commit: don't use zero-filled arrays in
tests, they mask a bug
Posted by bo...@apache.org.
don't use zero-filled arrays in tests, they mask a bug
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/62202d2a
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/62202d2a
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/62202d2a
Branch: refs/heads/master
Commit: 62202d2acd00b43415ce9ed45e1c37f42d6ef616
Parents: 47213fe
Author: Stefan Bodewig <bo...@apache.org>
Authored: Thu Apr 6 10:59:51 2017 +0200
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Thu Apr 6 10:59:51 2017 +0200
----------------------------------------------------------------------
.../lz4/BlockLZ4CompressorOutputStreamTest.java | 43 ++++++++++++++------
1 file changed, 30 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/62202d2a/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStreamTest.java
index 59956e1..45bf684 100644
--- a/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStreamTest.java
@@ -194,7 +194,7 @@ public class BlockLZ4CompressorOutputStreamTest {
// though. (4 is the minimum length for a back-reference
// in LZ4
byte[] compressed = compress(i);
- byte[] expected = new byte[i + 1];
+ byte[] expected = prepareExpected(i + 1);
expected[0] = (byte) (i<<4);
Assert.assertArrayEquals("input length is " + i, expected, compressed);
}
@@ -208,7 +208,7 @@ public class BlockLZ4CompressorOutputStreamTest {
// twelve byte literal trailer and the back-reference
// would fall below the minimal size
byte[] compressed = compress(i);
- byte[] expected = i < 15 ? new byte[i + 1] : new byte[i + 2];
+ byte[] expected = prepareExpected(i < 15 ? i + 1 : i + 2);
if (i < 15) {
expected[0] = (byte) (i<<4);
} else {
@@ -224,9 +224,11 @@ public class BlockLZ4CompressorOutputStreamTest {
// this time even our algorithm is willing to break up the
// back-reference
byte[] compressed = compress(i);
- byte[] expected = new byte[17];
+ byte[] expected = prepareExpected(17);
expected[0] = (byte) ((1<<4) | i - 17);
- expected[2] = 1; // offset
+ // two-byte offset
+ expected[2] = 1;
+ expected[3] = 0;
expected[4] = (byte) (12<<4);
Assert.assertArrayEquals("input length is " + i, expected, compressed);
}
@@ -240,9 +242,11 @@ public class BlockLZ4CompressorOutputStreamTest {
// literal of length i
// we can split the back-reference and merge it with the literal
byte[] compressed = compress(16, i);
- byte[] expected = new byte[17];
+ byte[] expected = prepareExpected(17);
expected[0] = (byte) ((1<<4) | i - 1);
- expected[2] = 1; // offset
+ // two-byte offset
+ expected[2] = 1;
+ expected[3] = 0;
expected[4] = (byte) (12<<4);
for (int j = 0; j < i; j++) {
expected[expected.length - 1 - j] = 1;
@@ -258,9 +262,11 @@ public class BlockLZ4CompressorOutputStreamTest {
// requirements by just rewriting the last Pair, but our
// algorithm will chip off a few bytes from the first Pair
byte[] compressed = compress(16, i);
- byte[] expected = new byte[17];
+ byte[] expected = prepareExpected(17);
expected[0] = (byte) ((1<<4) | i - 1);
- expected[2] = 1; // offset
+ // two-byte offset
+ expected[2] = 1;
+ expected[3] = 0;
expected[4] = (byte) (12<<4);
for (int j = 0; j < i; j++) {
expected[expected.length - 1 - j] = 1;
@@ -275,9 +281,11 @@ public class BlockLZ4CompressorOutputStreamTest {
// this shouldn't affect the first pair at all as
// rewriting the second one is sufficient
byte[] compressed = compress(16, i);
- byte[] expected = new byte[i + 5];
+ byte[] expected = prepareExpected(i + 5);
expected[0] = (byte) ((1<<4) | 11);
- expected[2] = 1; // offset
+ // two-byte offset
+ expected[2] = 1;
+ expected[3] = 0;
expected[4] = (byte) (i<<4);
for (int j = 0; j < i; j++) {
expected[expected.length - 1 - j] = 1;
@@ -295,9 +303,11 @@ public class BlockLZ4CompressorOutputStreamTest {
// literal and one byte is chopped off of the first pair's
// back-reference
byte[] compressed = compress(6, 5, 5, 1);
- byte[] expected = new byte[17];
+ byte[] expected = prepareExpected(17);
expected[0] = (byte) (1<<4);
- expected[2] = 1; // offset
+ // two-byte offset
+ expected[2] = 1;
+ expected[3] = 0;
expected[4] = (byte) (12<<4);
for (int i = 6; i < 11; i++) {
expected[i] = 1;
@@ -314,7 +324,8 @@ public class BlockLZ4CompressorOutputStreamTest {
}
private byte[] compress(int lengthBeforeTrailer, int... lengthOfTrailers) throws IOException {
- return compress(new byte[lengthBeforeTrailer], lengthOfTrailers);
+ byte[] b = prepareExpected(lengthBeforeTrailer);
+ return compress(b, lengthOfTrailers);
}
private byte[] compress(byte[] input, int... lengthOfTrailers) throws IOException {
@@ -331,4 +342,10 @@ public class BlockLZ4CompressorOutputStreamTest {
return baos.toByteArray();
}
}
+
+ private byte[] prepareExpected(int length) {
+ byte[] b = new byte[length];
+ Arrays.fill(b, (byte) -1);
+ return b;
+ }
}