You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/01/14 14:12:30 UTC
[2/2] commons-compress git commit: extract little endian formatting
extract little endian formatting
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/75ba40cd
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/75ba40cd
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/75ba40cd
Branch: refs/heads/master
Commit: 75ba40cd9236761b4b3e49013eb38f754b4e539b
Parents: 7ba02aa
Author: Stefan Bodewig <bo...@apache.org>
Authored: Sat Jan 14 15:12:06 2017 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Sat Jan 14 15:12:06 2017 +0100
----------------------------------------------------------------------
.../FramedSnappyCompressorInputStream.java | 25 ++-
.../FramedSnappyCompressorOutputStream.java | 9 +-
.../snappy/SnappyCompressorInputStream.java | 28 ++-
.../snappy/SnappyCompressorOutputStream.java | 8 +-
.../commons/compress/utils/ByteUtils.java | 176 +++++++++++++++++++
5 files changed, 208 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/75ba40cd/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
index 7b292d3..0f262da 100644
--- a/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.utils.BoundedInputStream;
+import org.apache.commons.compress.utils.ByteUtils;
import org.apache.commons.compress.utils.IOUtils;
/**
@@ -75,6 +76,13 @@ public class FramedSnappyCompressorInputStream extends CompressorInputStream {
private final int blockSize;
private final PureJavaCrc32C checksum = new PureJavaCrc32C();
+ private final ByteUtils.ByteSupplier supplier = new ByteUtils.ByteSupplier() {
+ @Override
+ public int getAsByte() throws IOException {
+ return readOneByte();
+ }
+ };
+
/**
* Constructs a new input stream that decompresses
* snappy-framed-compressed data from the specified input stream
@@ -246,11 +254,7 @@ public class FramedSnappyCompressorInputStream extends CompressorInputStream {
if (read != 4) {
throw new IOException("premature end of stream");
}
- long crc = 0;
- for (int i = 0; i < 4; i++) {
- crc |= (b[i] & 0xFFL) << (8 * i);
- }
- return crc;
+ return ByteUtils.fromLittleEndian(b);
}
static long unmask(long x) {
@@ -262,16 +266,7 @@ public class FramedSnappyCompressorInputStream extends CompressorInputStream {
}
private int readSize() throws IOException {
- int b = 0;
- int sz = 0;
- for (int i = 0; i < 3; i++) {
- b = readOneByte();
- if (b == -1) {
- throw new IOException("premature end of stream");
- }
- sz |= (b << (i * 8));
- }
- return sz;
+ return (int) ByteUtils.fromLittleEndian(supplier, 3);
}
private void skipBlock() throws IOException {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/75ba40cd/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
index 3d20d0f..e4d2a5e 100644
--- a/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.io.OutputStream;
import org.apache.commons.compress.compressors.CompressorOutputStream;
+import org.apache.commons.compress.utils.ByteUtils;
/**
* CompressorOutputStream for the framing Snappy format.
@@ -47,6 +48,8 @@ public class FramedSnappyCompressorOutputStream extends CompressorOutputStream {
private final byte[] buffer = new byte[MAX_COMPRESSED_BUFFER_SIZE];
private int currentIndex = 0;
+ private final ByteUtils.ByteConsumer consumer;
+
/**
* Constructs a new output stream that compresses
* snappy-framed-compressed data to the specified output stream.
@@ -55,6 +58,7 @@ public class FramedSnappyCompressorOutputStream extends CompressorOutputStream {
*/
public FramedSnappyCompressorOutputStream(final OutputStream out) throws IOException {
this.out = out;
+ consumer = new ByteUtils.OutputStreamByteConsumer(out);
out.write(FramedSnappyCompressorInputStream.SZ_SIGNATURE);
}
@@ -111,10 +115,7 @@ public class FramedSnappyCompressorOutputStream extends CompressorOutputStream {
}
private void writeLittleEndian(final int numBytes, long num) throws IOException {
- for (int i = 0; i < numBytes; i++) {
- out.write((int) (num & 0xff));
- num >>= 8;
- }
+ ByteUtils.toLittleEndian(consumer, num, numBytes);
}
private void writeCrc() throws IOException {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/75ba40cd/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorInputStream.java
index 237809f..6b1f1d8 100644
--- a/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorInputStream.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.InputStream;
import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.utils.ByteUtils;
import org.apache.commons.compress.utils.IOUtils;
/**
@@ -72,6 +73,13 @@ public class SnappyCompressorInputStream extends CompressorInputStream {
private boolean endReached = false;
+ private final ByteUtils.ByteSupplier supplier = new ByteUtils.ByteSupplier() {
+ @Override
+ public int getAsByte() throws IOException {
+ return readOneByte();
+ }
+ };
+
/**
* Constructor using the default buffer size of 32k.
*
@@ -209,8 +217,7 @@ public class SnappyCompressorInputStream extends CompressorInputStream {
length = (b >> 2) + 1;
uncompressedBytesRemaining -= length;
- offset = readOneByte();
- offset |= readOneByte() << 8;
+ offset = ByteUtils.fromLittleEndian(supplier, 2);
if (expandCopy(offset, length)) {
return;
@@ -229,10 +236,7 @@ public class SnappyCompressorInputStream extends CompressorInputStream {
length = (b >> 2) + 1;
uncompressedBytesRemaining -= length;
- offset = readOneByte();
- offset |= readOneByte() << 8;
- offset |= readOneByte() << 16;
- offset |= ((long) readOneByte()) << 24;
+ offset = ByteUtils.fromLittleEndian(supplier, 4);
if (expandCopy(offset, length)) {
return;
@@ -275,19 +279,13 @@ public class SnappyCompressorInputStream extends CompressorInputStream {
length = readOneByte();
break;
case 61:
- length = readOneByte();
- length |= readOneByte() << 8;
+ length = (int) ByteUtils.fromLittleEndian(supplier, 2);
break;
case 62:
- length = readOneByte();
- length |= readOneByte() << 8;
- length |= readOneByte() << 16;
+ length = (int) ByteUtils.fromLittleEndian(supplier, 3);
break;
case 63:
- length = readOneByte();
- length |= readOneByte() << 8;
- length |= readOneByte() << 16;
- length |= (((long) readOneByte()) << 24);
+ length = (int) ByteUtils.fromLittleEndian(supplier, 4);
break;
default:
length = b >> 2;
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/75ba40cd/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorOutputStream.java
index 0eb1588..8827aeb 100644
--- a/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorOutputStream.java
@@ -24,6 +24,7 @@ import java.io.OutputStream;
import org.apache.commons.compress.compressors.CompressorOutputStream;
import org.apache.commons.compress.compressors.lz77support.LZ77Compressor;
import org.apache.commons.compress.compressors.lz77support.Parameters;
+import org.apache.commons.compress.utils.ByteUtils;
/**
* CompressorOutputStream for the raw Snappy format.
@@ -53,6 +54,7 @@ import org.apache.commons.compress.compressors.lz77support.Parameters;
public class SnappyCompressorOutputStream extends CompressorOutputStream {
private final LZ77Compressor compressor;
private final OutputStream os;
+ private final ByteUtils.ByteConsumer consumer;
// used in one-arg write method
private final byte[] oneByte = new byte[1];
@@ -96,6 +98,7 @@ public class SnappyCompressorOutputStream extends CompressorOutputStream {
public SnappyCompressorOutputStream(final OutputStream os, final long uncompressedSize, Parameters params)
throws IOException {
this.os = os;
+ consumer = new ByteUtils.OutputStreamByteConsumer(os);
compressor = new LZ77Compressor(params, new LZ77Compressor.Callback() {
public void accept(LZ77Compressor.Block block) throws IOException {
//System.err.println(block);
@@ -206,10 +209,7 @@ public class SnappyCompressorOutputStream extends CompressorOutputStream {
}
private void writeLittleEndian(final int numBytes, int num) throws IOException {
- for (int i = 0; i < numBytes; i++) {
- os.write(num & 0xff);
- num >>= 8;
- }
+ ByteUtils.toLittleEndian(consumer, num, numBytes);
}
// Back-references ("copies") have their offset/size information
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/75ba40cd/src/main/java/org/apache/commons/compress/utils/ByteUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/utils/ByteUtils.java b/src/main/java/org/apache/commons/compress/utils/ByteUtils.java
new file mode 100644
index 0000000..405a0e3
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/utils/ByteUtils.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.commons.compress.utils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Utility methods for reading and writing bytes.
+ * @since 1.14
+ */
+public final class ByteUtils {
+ private ByteUtils() { /* no instances */ }
+
+ /**
+ * Used to supply bytes.
+ * @since 1.14
+ */
+ public interface ByteSupplier {
+ /**
+ * The contract is similar to {@link InputStream#read()}, return
+ * the byte as an unsigned int, -1 if there are no more bytes.
+ */
+ int getAsByte() throws IOException;
+ }
+
+ /**
+ * Used to consume bytes.
+ * @since 1.14
+ */
+ public interface ByteConsumer {
+ /**
+ * The contract is similar to {@link OutputStream#write(int)},
+ * consume the lower eight bytes of the int as a byte.
+ */
+ void accept(int b) throws IOException;
+ }
+
+ /**
+ * Reads the given byte array as a little endian long.
+ * @param bytes the byte array to convert
+ */
+ public static long fromLittleEndian(byte[] bytes) {
+ return fromLittleEndian(bytes, 0, bytes.length);
+ }
+
+ /**
+ * Reads the given byte array as a little endian long.
+ * @param bytes the byte array to convert
+ * @param off the offset into the array that starts the value
+ * @param length the number of bytes representing the value
+ * @throws IllegalArgumentException if len is bigger than eight
+ */
+ public static long fromLittleEndian(byte[] bytes, final int off, final int length) {
+ if (length > 8) {
+ throw new IllegalArgumentException("can't read more than eight bytes into a long value");
+ }
+ long l = 0;
+ for (int i = 0; i < length; i++) {
+ l |= (bytes[off + i] & 0xffl) << (8 * i);
+ }
+ return l;
+ }
+
+ /**
+ * Reads the given number of bytes from the given stream as a little endian long.
+ * @param in the stream to read from
+ * @param length the number of bytes representing the value
+ * @throws IllegalArgumentException if len is bigger than eight
+ * @throws IOException if reading fails or the stream doesn't
+ * contain the given number of bytes anymore
+ */
+ public static long fromLittleEndian(InputStream in, int length) throws IOException {
+ return fromLittleEndian(new InputStreamByteSupplier(in), length);
+ }
+
+ /**
+ * Reads the given number of bytes from the given supplier as a little endian long.
+ * @param supplier the supplier for bytes
+ * @param length the number of bytes representing the value
+ * @throws IllegalArgumentException if len is bigger than eight
+ * @throws IOException if the supplier fails or doesn't supply the
+ * given number of bytes anymore
+ */
+ public static long fromLittleEndian(ByteSupplier supplier, final int length) throws IOException {
+ if (length > 8) {
+ throw new IllegalArgumentException("can't read more than eight bytes into a long value");
+ }
+ long l = 0;
+ for (int i = 0; i < length; i++) {
+ int b = supplier.getAsByte();
+ if (b == -1) {
+ throw new IOException("premature end of data");
+ }
+ l |= (b << (i * 8));
+ }
+ return l;
+ }
+
+ /**
+ * Writes the given value to the given stream as a little endian
+ * array of the given length.
+ * @param out the stream to write to
+ * @param value the value to write
+ * @param length the number of bytes to use to represent the value
+ * @throws IOException if writing fails
+ */
+ public static void toLittleEndian(OutputStream out, final long value, final int length)
+ throws IOException {
+ toLittleEndian(new OutputStreamByteConsumer(out), value, length);
+ }
+
+ /**
+ * Provides the given value to the given consumer as a little endian
+ * sequence of the given length.
+ * @param consumer the consumer to provide the bytes to
+ * @param value the value to provide
+ * @param length the number of bytes to use to represent the value
+ * @throws IOException if writing fails
+ */
+ public static void toLittleEndian(ByteConsumer consumer, final long value, final int length)
+ throws IOException {
+ long num = value;
+ for (int i = 0; i < length; i++) {
+ consumer.accept((int) (num & 0xff));
+ num >>= 8;
+ }
+ }
+
+ /**
+ * {@link ByteSupplier} based on {@link InputStream}.
+ * @since 1.14
+ */
+ public static class InputStreamByteSupplier implements ByteSupplier {
+ private final InputStream is;
+ public InputStreamByteSupplier(InputStream is) {
+ this.is = is;
+ }
+ @Override
+ public int getAsByte() throws IOException {
+ return is.read();
+ }
+ }
+
+ /**
+ * {@link ByteConsumer} based on {@link OutputStream}.
+ * @since 1.14
+ */
+ public static class OutputStreamByteConsumer implements ByteConsumer {
+ private final OutputStream os;
+ public OutputStreamByteConsumer(OutputStream os) {
+ this.os = os;
+ }
+ @Override
+ public void accept(int b) throws IOException {
+ os.write(b);
+ }
+ }
+}