You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by bl...@apache.org on 2015/04/28 01:12:15 UTC
[18/51] [partial] parquet-mr git commit: PARQUET-23: Rename to
org.apache.parquet.
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BitPacking.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BitPacking.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BitPacking.java
new file mode 100644
index 0000000..c0acd11
--- /dev/null
+++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BitPacking.java
@@ -0,0 +1,725 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingReader;
+import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingWriter;
+
+// TODO: rework the whole thing. It does not need to use streams at all
+/**
+ * provides the correct implementation of a bitpacking based on the width in bits
+ *
+ * @author Julien Le Dem
+ *
+ */
+public class BitPacking {
+
+ /**
+ * to writes ints to a stream packed to only the needed bits.
+ * there is no guarantee of corecteness if ints larger than the max size are written
+ *
+ * @author Julien Le Dem
+ *
+ */
+ abstract public static class BitPackingWriter {
+ /**
+ * will write the bits to the underlying stream aligned on the buffer size
+ * @param val the value to encode
+ * @throws IOException
+ */
+ abstract public void write(int val) throws IOException;
+
+ /**
+ * will flush the buffer to the underlying stream (and pad with 0s)
+ * @throws IOException
+ */
+ abstract public void finish() throws IOException;
+ }
+
+ /**
+ * to read back what has been written with the corresponding writer
+ *
+ * @author Julien Le Dem
+ *
+ */
+ abstract public static class BitPackingReader {
+
+ /**
+ *
+ * @return and int decoded from the underlying stream
+ * @throws IOException
+ */
+ abstract public int read() throws IOException;
+ }
+
+ private BitPacking() {
+ }
+
+ /**
+ * @param bitLength the width in bits of the integers to write
+ * @param out the stream to write the bytes to
+ * @return the correct implementation for the width
+ */
+ public static BitPackingWriter getBitPackingWriter(int bitLength, OutputStream out) {
+ switch (bitLength) {
+ case 0:
+ return new ZeroBitPackingWriter();
+ case 1:
+ return new OneBitPackingWriter(out);
+ case 2:
+ return new TwoBitPackingWriter(out);
+ case 3:
+ return new ThreeBitPackingWriter(out);
+ case 4:
+ return new FourBitPackingWriter(out);
+ case 5:
+ return new FiveBitPackingWriter(out);
+ case 6:
+ return new SixBitPackingWriter(out);
+ case 7:
+ return new SevenBitPackingWriter(out);
+ case 8:
+ return new EightBitPackingWriter(out);
+ default:
+ throw new UnsupportedOperationException("only support up to 8 for now");
+ }
+ }
+
+ /**
+ *
+ * @param bitLength the width in bits of the integers to read
+ * @param inthe stream to read the bytes from
+ * @return the correct implementation for the width
+ */
+ public static BitPackingReader createBitPackingReader(int bitLength, InputStream in, long valueCount) {
+ switch (bitLength) {
+ case 0:
+ return new ZeroBitPackingReader();
+ case 1:
+ return new OneBitPackingReader(in);
+ case 2:
+ return new TwoBitPackingReader(in);
+ case 3:
+ return new ThreeBitPackingReader(in, valueCount);
+ case 4:
+ return new FourBitPackingReader(in);
+ case 5:
+ return new FiveBitPackingReader(in, valueCount);
+ case 6:
+ return new SixBitPackingReader(in, valueCount);
+ case 7:
+ return new SevenBitPackingReader(in, valueCount);
+ case 8:
+ return new EightBitPackingReader(in);
+ default:
+ throw new UnsupportedOperationException("only support up to 8 for now");
+ }
+ }
+}
+
+abstract class BaseBitPackingWriter extends BitPackingWriter {
+
+ void finish(int numberOfBits, int buffer, OutputStream out) throws IOException {
+ int padding = numberOfBits % 8 == 0 ? 0 : 8 - (numberOfBits % 8);
+ buffer = buffer << padding;
+ int numberOfBytes = (numberOfBits + padding) / 8;
+ for (int i = (numberOfBytes - 1) * 8; i >= 0 ; i -= 8) {
+ out.write((buffer >>> i) & 0xFF);
+ }
+ }
+
+ void finish(int numberOfBits, long buffer, OutputStream out) throws IOException {
+ int padding = numberOfBits % 8 == 0 ? 0 : 8 - (numberOfBits % 8);
+ buffer = buffer << padding;
+ int numberOfBytes = (numberOfBits + padding) / 8;
+ for (int i = (numberOfBytes - 1) * 8; i >= 0 ; i -= 8) {
+ out.write((int)(buffer >>> i) & 0xFF);
+ }
+ }
+}
+abstract class BaseBitPackingReader extends BitPackingReader {
+
+ int alignToBytes(int bitsCount) {
+ return BytesUtils.paddedByteCountFromBits(bitsCount);
+ }
+
+}
+
+class ZeroBitPackingWriter extends BitPackingWriter {
+
+ @Override
+ public void write(int val) throws IOException {
+ }
+
+ @Override
+ public void finish() {
+ }
+
+}
+class ZeroBitPackingReader extends BitPackingReader {
+
+ @Override
+ public int read() throws IOException {
+ return 0;
+ }
+
+}
+
+class OneBitPackingWriter extends BitPackingWriter {
+
+ private OutputStream out;
+
+ private int buffer = 0;
+ private int count = 0;
+
+ public OneBitPackingWriter(OutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public void write(int val) throws IOException {
+ buffer = buffer << 1;
+ buffer |= val;
+ ++ count;
+ if (count == 8) {
+ out.write(buffer);
+ buffer = 0;
+ count = 0;
+ }
+ }
+
+ @Override
+ public void finish() throws IOException {
+ while (count != 0) {
+ write(0);
+ }
+ // check this does not impede perf
+ out = null;
+ }
+
+}
+class OneBitPackingReader extends BitPackingReader {
+
+ private final InputStream in;
+
+ private int buffer = 0;
+ private int count = 0;
+
+ public OneBitPackingReader(InputStream in) {
+ this.in = in;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (count == 0) {
+ buffer = in.read();
+ count = 8;
+ }
+ int result = (buffer >> (count - 1)) & 1;
+ -- count;
+ return result;
+ }
+
+}
+
+class TwoBitPackingWriter extends BitPackingWriter {
+
+ private OutputStream out;
+
+ private int buffer = 0;
+ private int count = 0;
+
+ public TwoBitPackingWriter(OutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public void write(int val) throws IOException {
+ buffer = buffer << 2;
+ buffer |= val;
+ ++ count;
+ if (count == 4) {
+ out.write(buffer);
+ buffer = 0;
+ count = 0;
+ }
+ }
+
+ @Override
+ public void finish() throws IOException {
+ while (count != 0) {
+ write(0);
+ }
+ // check this does not impede perf
+ out = null;
+ }
+
+}
+class TwoBitPackingReader extends BitPackingReader {
+
+ private final InputStream in;
+
+ private int buffer = 0;
+ private int count = 0;
+
+ public TwoBitPackingReader(InputStream in) {
+ this.in = in;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (count == 0) {
+ buffer = in.read();
+ count = 4;
+ }
+ int result = (buffer >> ((count - 1) * 2)) & 3;
+ -- count;
+ return result;
+ }
+
+}
+
+class ThreeBitPackingWriter extends BaseBitPackingWriter {
+
+ private OutputStream out;
+
+ private int buffer = 0;
+ private int count = 0;
+
+ public ThreeBitPackingWriter(OutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public void write(int val) throws IOException {
+ buffer = buffer << 3;
+ buffer |= val;
+ ++ count;
+ if (count == 8) {
+ out.write((buffer >>> 16) & 0xFF);
+ out.write((buffer >>> 8) & 0xFF);
+ out.write((buffer >>> 0) & 0xFF);
+ buffer = 0;
+ count = 0;
+ }
+ }
+
+ @Override
+ public void finish() throws IOException {
+ if (count != 0) {
+ int numberOfBits = count * 3;
+ finish(numberOfBits, buffer, out);
+ buffer = 0;
+ count = 0;
+ }
+ // check this does not impede perf
+ out = null;
+ }
+
+}
+class ThreeBitPackingReader extends BaseBitPackingReader {
+
+ private final InputStream in;
+ private final long valueCount;
+
+ private int buffer = 0;
+ private int count = 0;
+
+ private long totalRead = 0;
+
+ public ThreeBitPackingReader(InputStream in, long valueCount) {
+ this.in = in;
+ this.valueCount = valueCount;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (count == 0) {
+ if (valueCount - totalRead < 8) {
+ buffer = 0;
+ int bitsToRead = 3 * (int)(valueCount - totalRead);
+ int bytesToRead = alignToBytes(bitsToRead);
+ for (int i = 3 - 1 ; i >= 3 - bytesToRead ; i--) {
+ buffer |= in.read() << (i * 8);
+ }
+ count = 8;
+ totalRead = valueCount;
+ } else {
+ buffer = (in.read() << 16) + (in.read() << 8) + in.read();
+ count = 8;
+ totalRead += 8;
+ }
+ }
+ int result = (buffer >> ((count - 1) * 3)) & 7;
+ -- count;
+ return result;
+ }
+
+}
+
+class FourBitPackingWriter extends BitPackingWriter {
+
+ private OutputStream out;
+
+ private int buffer = 0;
+ private int count = 0;
+
+ public FourBitPackingWriter(OutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public void write(int val) throws IOException {
+ buffer = buffer << 4;
+ buffer |= val;
+ ++ count;
+ if (count == 2) {
+ out.write(buffer);
+ buffer = 0;
+ count = 0;
+ }
+ }
+
+ @Override
+ public void finish() throws IOException {
+ while (count != 0) {
+ // downside: this aligns on whatever the buffer size is.
+ write(0);
+ }
+ // check this does not impede perf
+ out = null;
+ }
+
+}
+class FourBitPackingReader extends BitPackingReader {
+
+ private final InputStream in;
+
+ private int buffer = 0;
+ private int count = 0;
+
+ public FourBitPackingReader(InputStream in) {
+ this.in = in;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (count == 0) {
+ buffer = in.read();
+ count = 2;
+ }
+ int result = (buffer >> ((count - 1) * 4)) & 15;
+ -- count;
+ return result;
+ }
+
+}
+
+class FiveBitPackingWriter extends BaseBitPackingWriter {
+
+ private OutputStream out;
+
+ private long buffer = 0;
+ private int count = 0;
+
+ public FiveBitPackingWriter(OutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public void write(int val) throws IOException {
+ buffer = buffer << 5;
+ buffer |= val;
+ ++ count;
+ if (count == 8) {
+ out.write((int)(buffer >>> 32) & 0xFF);
+ out.write((int)(buffer >>> 24) & 0xFF);
+ out.write((int)(buffer >>> 16) & 0xFF);
+ out.write((int)(buffer >>> 8) & 0xFF);
+ out.write((int)(buffer >>> 0) & 0xFF);
+ buffer = 0;
+ count = 0;
+ }
+ }
+
+ @Override
+ public void finish() throws IOException {
+ if (count != 0) {
+ int numberOfBits = count * 5;
+ finish(numberOfBits, buffer, out);
+ buffer = 0;
+ count = 0;
+ }
+ // check this does not impede perf
+ out = null;
+ }
+
+}
+class FiveBitPackingReader extends BaseBitPackingReader {
+
+ private final InputStream in;
+ private final long valueCount;
+
+ private long buffer = 0;
+ private int count = 0;
+ private long totalRead = 0;
+
+
+ public FiveBitPackingReader(InputStream in, long valueCount) {
+ this.in = in;
+ this.valueCount = valueCount;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (count == 0) {
+ if (valueCount - totalRead < 8) {
+ buffer = 0;
+ int bitsToRead = 5 * (int)(valueCount - totalRead);
+ int bytesToRead = alignToBytes(bitsToRead);
+ for (int i = 5 - 1; i >= 5 - bytesToRead ; i--) {
+ buffer |= (((long)in.read()) & 255) << (i * 8);
+ }
+ count = 8;
+ totalRead = valueCount;
+ } else {
+ buffer =
+ ((((long)in.read()) & 255) << 32)
+ + ((((long)in.read()) & 255) << 24)
+ + (in.read() << 16)
+ + (in.read() << 8)
+ + in.read();
+ count = 8;
+ totalRead += 8;
+ }
+ }
+ int result = (((int)(buffer >> ((count - 1) * 5))) & 31);
+ -- count;
+ return result;
+ }
+
+}
+
+class SixBitPackingWriter extends BaseBitPackingWriter {
+
+ private OutputStream out;
+
+ private int buffer = 0;
+ private int count = 0;
+
+ public SixBitPackingWriter(OutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public void write(int val) throws IOException {
+ buffer = buffer << 6;
+ buffer |= val;
+ ++ count;
+ if (count == 4) {
+ out.write((buffer >>> 16) & 0xFF);
+ out.write((buffer >>> 8) & 0xFF);
+ out.write((buffer >>> 0) & 0xFF);
+ buffer = 0;
+ count = 0;
+ }
+ }
+
+ @Override
+ public void finish() throws IOException {
+ if (count != 0) {
+ int numberOfBits = count * 6;
+ finish(numberOfBits, buffer, out);
+ buffer = 0;
+ count = 0;
+ }
+ // check this does not impede perf
+ out = null;
+ }
+
+}
+class SixBitPackingReader extends BaseBitPackingReader {
+
+ private final InputStream in;
+ private final long valueCount;
+
+ private int buffer = 0;
+ private int count = 0;
+
+ private long totalRead = 0;
+
+
+ public SixBitPackingReader(InputStream in, long valueCount) {
+ this.in = in;
+ this.valueCount = valueCount;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (count == 0) {
+ if (valueCount - totalRead < 4) {
+ buffer = 0;
+ int bitsToRead = 6 * (int)(valueCount - totalRead);
+ int bytesToRead = alignToBytes(bitsToRead);
+ for (int i = 3 - 1; i >= 3 - bytesToRead ; i--) {
+ buffer |= in.read() << (i * 8);
+ }
+ count = 4;
+ totalRead = valueCount;
+ } else {
+ buffer = (in.read() << 16) + (in.read() << 8) + in.read();
+ count = 4;
+ totalRead += 4;
+ }
+ }
+ int result = (buffer >> ((count - 1) * 6)) & 63;
+ -- count;
+ return result;
+ }
+
+}
+
+class SevenBitPackingWriter extends BaseBitPackingWriter {
+
+ private OutputStream out;
+
+ private long buffer = 0;
+ private int count = 0;
+
+ public SevenBitPackingWriter(OutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public void write(int val) throws IOException {
+ buffer = buffer << 7;
+ buffer |= val;
+ ++ count;
+ if (count == 8) {
+ out.write((int)(buffer >>> 48) & 0xFF);
+ out.write((int)(buffer >>> 40) & 0xFF);
+ out.write((int)(buffer >>> 32) & 0xFF);
+ out.write((int)(buffer >>> 24) & 0xFF);
+ out.write((int)(buffer >>> 16) & 0xFF);
+ out.write((int)(buffer >>> 8) & 0xFF);
+ out.write((int)(buffer >>> 0) & 0xFF);
+ buffer = 0;
+ count = 0;
+ }
+ }
+
+ @Override
+ public void finish() throws IOException {
+ if (count != 0) {
+ int numberOfBits = count * 7;
+ finish(numberOfBits, buffer, out);
+ buffer = 0;
+ count = 0;
+ }
+ // check this does not impede perf
+ out = null;
+ }
+
+}
+class SevenBitPackingReader extends BaseBitPackingReader {
+
+ private final InputStream in;
+ private final long valueCount;
+
+ private long buffer = 0;
+ private int count = 0;
+ private long totalRead = 0;
+
+
+ public SevenBitPackingReader(InputStream in, long valueCount) {
+ this.in = in;
+ this.valueCount = valueCount;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (count == 0) {
+ if (valueCount - totalRead < 8) {
+ buffer = 0;
+ int bitsToRead = 7 * (int)(valueCount - totalRead);
+ int bytesToRead = alignToBytes(bitsToRead);
+ for (int i = 7 - 1; i >= 7 - bytesToRead ; i--) {
+ buffer |= (((long)in.read()) & 255) << (i * 8);
+ }
+ count = 8;
+ totalRead = valueCount;
+ } else {
+ buffer =
+ ((((long)in.read()) & 255) << 48)
+ + ((((long)in.read()) & 255) << 40)
+ + ((((long)in.read()) & 255) << 32)
+ + ((((long)in.read()) & 255) << 24)
+ + (in.read() << 16)
+ + (in.read() << 8)
+ + in.read();
+ count = 8;
+ totalRead += 8;
+ }
+ }
+ int result = (((int)(buffer >> ((count - 1) * 7))) & 127);
+ -- count;
+ return result;
+ }
+
+}
+
+class EightBitPackingWriter extends BitPackingWriter {
+
+ private OutputStream out;
+
+ public EightBitPackingWriter(OutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public void write(int val) throws IOException {
+ out.write(val);
+ }
+
+ @Override
+ public void finish() throws IOException {
+ // check this does not impede perf
+ out = null;
+ }
+
+}
+class EightBitPackingReader extends BitPackingReader {
+
+ private final InputStream in;
+
+ public EightBitPackingReader(InputStream in) {
+ this.in = in;
+ }
+
+ @Override
+ public int read() throws IOException {
+ return in.read();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java
new file mode 100644
index 0000000..448c0be
--- /dev/null
+++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+import static org.apache.parquet.Log.DEBUG;
+import static org.apache.parquet.bytes.BytesInput.concat;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.parquet.Log;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.BytesUtils;
+
+/**
+ * Uses the generated Byte based bit packing to write ints into a BytesInput
+ *
+ * @author Julien Le Dem
+ *
+ */
+public class ByteBasedBitPackingEncoder {
+ private static final Log LOG = Log.getLog(ByteBasedBitPackingEncoder.class);
+
+ private static final int VALUES_WRITTEN_AT_A_TIME = 8;
+
+ private final int bitWidth;
+ private final BytePacker packer;
+ private final int[] input = new int[VALUES_WRITTEN_AT_A_TIME];
+ private final int slabSize;
+ private int inputSize;
+ private byte[] packed;
+ private int packedPosition;
+ private final List<BytesInput> slabs = new ArrayList<BytesInput>();
+ private int totalValues;
+
+ /**
+ * @param bitWidth the number of bits used to encode an int
+ */
+ public ByteBasedBitPackingEncoder(int bitWidth, Packer packer) {
+ this.bitWidth = bitWidth;
+ this.inputSize = 0;
+ // must be a multiple of bitWidth
+ this.slabSize = bitWidth * 64 * 1024;
+ initPackedSlab();
+ this.packer = packer.newBytePacker(bitWidth);
+ }
+
+ /**
+ * writes an int using the requested number of bits.
+ * accepts only value < 2^bitWidth
+ * @param value the value to write
+ * @throws IOException
+ */
+ public void writeInt(int value) throws IOException {
+ input[inputSize] = value;
+ ++ inputSize;
+ if (inputSize == VALUES_WRITTEN_AT_A_TIME) {
+ pack();
+ if (packedPosition == slabSize) {
+ slabs.add(BytesInput.from(packed));
+ initPackedSlab();
+ }
+ }
+ }
+
+ private void pack() {
+ packer.pack8Values(input, 0, packed, packedPosition);
+ packedPosition += bitWidth;
+ totalValues += inputSize;
+ inputSize = 0;
+ }
+
+ private void initPackedSlab() {
+ packed = new byte[slabSize];
+ packedPosition = 0;
+ }
+
+ /**
+ * @return the bytes representing the packed values
+ * @throws IOException
+ */
+ public BytesInput toBytes() throws IOException {
+ int packedByteLength = packedPosition + BytesUtils.paddedByteCountFromBits(inputSize * bitWidth);
+
+ if (DEBUG) LOG.debug("writing " + (slabs.size() * slabSize + packedByteLength) + " bytes");
+ if (inputSize > 0) {
+ for (int i = inputSize; i < input.length; i++) {
+ input[i] = 0;
+ }
+ pack();
+ }
+ return concat(concat(slabs), BytesInput.from(packed, 0, packedByteLength));
+ }
+
+ /**
+ * @return size of the data as it would be written
+ */
+ public long getBufferSize() {
+ return BytesUtils.paddedByteCountFromBits(totalValues * bitWidth);
+ }
+
+ /**
+ * @return total memory allocated
+ */
+ public long getAllocatedSize() {
+ return (slabs.size() * slabSize) + packed.length + input.length * 4;
+ }
+
+ public String memUsageString(String prefix) {
+ return String.format("%s ByteBitPacking %d slabs, %d bytes", prefix, slabs.size(), getAllocatedSize());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePacker.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePacker.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePacker.java
new file mode 100644
index 0000000..b9a37ad
--- /dev/null
+++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePacker.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+/**
+ * Packs and unpacks into bytes
+ *
+ * packing unpacking treats:
+ * - n values at a time (with n % 8 == 0)
+ * - bitWidth * (n/8) bytes at a time.
+ *
+ * @author Julien Le Dem
+ *
+ */
+public abstract class BytePacker {
+
+ private final int bitWidth;
+
+ BytePacker(int bitWidth) {
+ this.bitWidth = bitWidth;
+ }
+
+ /**
+ * @return the width in bits used for encoding, also how many bytes are packed/unpacked at a time by pack8Values/unpack8Values
+ */
+ public final int getBitWidth() {
+ return bitWidth;
+ }
+
+ /**
+ * pack 8 values from input at inPos into bitWidth bytes in output at outPos.
+ * nextPosition: inPos += 8; outPos += getBitWidth()
+ * @param input the input values
+ * @param inPos where to read from in input
+ * @param output the output bytes
+ * @param outPos where to write to in output
+ */
+ public abstract void pack8Values(final int[] input, final int inPos, final byte[] output, final int outPos);
+
+ /**
+ * pack 32 values from input at inPos into bitWidth * 4 bytes in output at outPos.
+ * nextPosition: inPos += 32; outPos += getBitWidth() * 4
+ * @param input the input values
+ * @param inPos where to read from in input
+ * @param output the output bytes
+ * @param outPos where to write to in output
+ */
+ public abstract void pack32Values(int[] input, int inPos, byte[] output, int outPos);
+
+ /**
+ * unpack bitWidth bytes from input at inPos into 8 values in output at outPos.
+ * nextPosition: inPos += getBitWidth(); outPos += 8
+ * @param input the input bytes
+ * @param inPos where to read from in input
+ * @param output the output values
+ * @param outPos where to write to in output
+ */
+ public abstract void unpack8Values(final byte[] input, final int inPos, final int[] output, final int outPos);
+
+ /**
+ * unpack bitWidth * 4 bytes from input at inPos into 32 values in output at outPos.
+ * nextPosition: inPos += getBitWidth() * 4; outPos += 32
+ * @param input the input bytes
+ * @param inPos where to read from in input
+ * @param output the output values
+ * @param outPos where to write to in output
+ */
+ public abstract void unpack32Values(byte[] input, int inPos, int[] output, int outPos);
+
+}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePackerFactory.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePackerFactory.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePackerFactory.java
new file mode 100644
index 0000000..fd75461
--- /dev/null
+++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePackerFactory.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+public interface BytePackerFactory {
+
+ BytePacker newBytePacker(int width);
+
+}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPacker.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPacker.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPacker.java
new file mode 100644
index 0000000..1d8d616
--- /dev/null
+++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPacker.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+/**
+ * Packs and unpacks into ints
+ *
+ * packing unpacking treats:
+ * - 32 values at a time
+ * - bitWidth ints at a time.
+ *
+ * @author Julien Le Dem
+ *
+ */
+public abstract class IntPacker {
+
+ private final int bitWidth;
+
+ IntPacker(int bitWidth) {
+ this.bitWidth = bitWidth;
+ }
+
+ /**
+ * @return the width in bits used for encoding, also how many ints are packed/unpacked at a time
+ */
+ public final int getBitWidth() {
+ return bitWidth;
+ }
+
+ /**
+ * pack 32 values from input at inPos into bitWidth ints in output at outPos.
+ * nextPosition: inPos += 32; outPos += getBitWidth()
+ * @param input the input values
+ * @param inPos where to read from in input
+ * @param output the output ints
+ * @param outPos where to write to in output
+ */
+ public abstract void pack32Values(int[] input, int inPos, int[] output, int outPos);
+
+ /**
+ * unpack bitWidth ints from input at inPos into 32 values in output at outPos.
+ * nextPosition: inPos += getBitWidth(); outPos += 32
+ * @param input the input int
+ * @param inPos where to read from in input
+ * @param output the output values
+ * @param outPos where to write to in output
+ */
+ public abstract void unpack32Values(int[] input, int inPos, int[] output, int outPos);
+
+}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPackerFactory.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPackerFactory.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPackerFactory.java
new file mode 100644
index 0000000..a9586bd
--- /dev/null
+++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPackerFactory.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+public interface IntPackerFactory {
+
+ IntPacker newIntPacker(int width);
+
+}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/Packer.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/Packer.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/Packer.java
new file mode 100644
index 0000000..ed14edf
--- /dev/null
+++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/Packer.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+/**
+ * Factory for packing implementations
+ *
+ * @author Julien Le Dem
+ *
+ */
+public enum Packer {
+
+ /**
+ * packers who fill the Least Significant Bit First
+ * int and byte packer have the same result on Big Endian architectures
+ */
+ BIG_ENDIAN {
+ @Override
+ public IntPacker newIntPacker(int width) {
+ return beIntPackerFactory.newIntPacker(width);
+ }
+ @Override
+ public BytePacker newBytePacker(int width) {
+ return beBytePackerFactory.newBytePacker(width);
+ }
+ },
+
+ /**
+ * packers who fill the Most Significant Bit first
+ * int and byte packer have the same result on Little Endian architectures
+ */
+ LITTLE_ENDIAN {
+ @Override
+ public IntPacker newIntPacker(int width) {
+ return leIntPackerFactory.newIntPacker(width);
+ }
+ @Override
+ public BytePacker newBytePacker(int width) {
+ return leBytePackerFactory.newBytePacker(width);
+ }
+ };
+
+ private static IntPackerFactory getIntPackerFactory(String name) {
+ return (IntPackerFactory)getStaticField("org.apache.parquet.column.values.bitpacking." + name, "factory");
+ }
+
+ private static BytePackerFactory getBytePackerFactory(String name) {
+ return (BytePackerFactory)getStaticField("org.apache.parquet.column.values.bitpacking." + name, "factory");
+ }
+
+ private static Object getStaticField(String className, String fieldName) {
+ try {
+ return Class.forName(className).getField(fieldName).get(null);
+ } catch (IllegalArgumentException e) {
+ throw new RuntimeException(e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ } catch (NoSuchFieldException e) {
+ throw new RuntimeException(e);
+ } catch (SecurityException e) {
+ throw new RuntimeException(e);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ static BytePackerFactory beBytePackerFactory = getBytePackerFactory("ByteBitPackingBE");
+ static IntPackerFactory beIntPackerFactory = getIntPackerFactory("LemireBitPackingBE");
+ static BytePackerFactory leBytePackerFactory = getBytePackerFactory("ByteBitPackingLE");
+ static IntPackerFactory leIntPackerFactory = getIntPackerFactory("LemireBitPackingLE");
+
+ /**
+ * @param width the width in bits of the packed values
+ * @return an int based packer
+ */
+ public abstract IntPacker newIntPacker(int width);
+
+ /**
+ * @param width the width in bits of the packed values
+ * @return a byte based packer
+ */
+ public abstract BytePacker newBytePacker(int width);
+}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/BytesInput.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/bytes/BytesInput.java b/parquet-encoding/src/main/java/parquet/bytes/BytesInput.java
deleted file mode 100644
index e0d25eb..0000000
--- a/parquet-encoding/src/main/java/parquet/bytes/BytesInput.java
+++ /dev/null
@@ -1,365 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.bytes;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Arrays;
-import java.util.List;
-
-import parquet.Log;
-
-
-/**
- * A source of bytes capable of writing itself to an output.
- * A BytesInput should be consumed right away.
- * It is not a container.
- * For example if it is referring to a stream,
- * subsequent BytesInput reads from the stream will be incorrect
- * if the previous has not been consumed.
- *
- * @author Julien Le Dem
- *
- */
-abstract public class BytesInput {
- private static final Log LOG = Log.getLog(BytesInput.class);
- private static final boolean DEBUG = false;//Log.DEBUG;
- private static final EmptyBytesInput EMPTY_BYTES_INPUT = new EmptyBytesInput();
-
- /**
- * logically concatenate the provided inputs
- * @param inputs the inputs to concatenate
- * @return a concatenated input
- */
- public static BytesInput concat(BytesInput... inputs) {
- return new SequenceBytesIn(Arrays.asList(inputs));
- }
-
- /**
- * logically concatenate the provided inputs
- * @param inputs the inputs to concatenate
- * @return a concatenated input
- */
- public static BytesInput concat(List<BytesInput> inputs) {
- return new SequenceBytesIn(inputs);
- }
-
- /**
- * @param in
- * @param bytes number of bytes to read
- * @return a BytesInput that will read that number of bytes from the stream
- */
- public static BytesInput from(InputStream in, int bytes) {
- return new StreamBytesInput(in, bytes);
- }
-
- /**
- *
- * @param in
- * @return a Bytes input that will write the given bytes
- */
- public static BytesInput from(byte[] in) {
- if (DEBUG) LOG.debug("BytesInput from array of " + in.length + " bytes");
- return new ByteArrayBytesInput(in, 0 , in.length);
- }
-
- public static BytesInput from(byte[] in, int offset, int length) {
- if (DEBUG) LOG.debug("BytesInput from array of " + length + " bytes");
- return new ByteArrayBytesInput(in, offset, length);
- }
-
- /**
- * @param intValue the int to write
- * @return a BytesInput that will write 4 bytes in little endian
- */
- public static BytesInput fromInt(int intValue) {
- return new IntBytesInput(intValue);
- }
-
- /**
- * @param intValue the int to write
- * @return a BytesInput that will write var int
- */
- public static BytesInput fromUnsignedVarInt(int intValue) {
- return new UnsignedVarIntBytesInput(intValue);
- }
-
- /**
- *
- * @param intValue the int to write
- */
- public static BytesInput fromZigZagVarInt(int intValue) {
- int zigZag = (intValue << 1) ^ (intValue >> 31);
- return new UnsignedVarIntBytesInput(zigZag);
- }
-
- /**
- * @param arrayOut
- * @return a BytesInput that will write the content of the buffer
- */
- public static BytesInput from(CapacityByteArrayOutputStream arrayOut) {
- return new CapacityBAOSBytesInput(arrayOut);
- }
-
- /**
- * @param arrayOut
- * @return a BytesInput that will write the content of the buffer
- */
- public static BytesInput from(ByteArrayOutputStream baos) {
- return new BAOSBytesInput(baos);
- }
-
- /**
- * @return an empty bytes input
- */
- public static BytesInput empty() {
- return EMPTY_BYTES_INPUT;
- }
-
- /**
- * copies the input into a new byte array
- * @param bytesInput
- * @return
- * @throws IOException
- */
- public static BytesInput copy(BytesInput bytesInput) throws IOException {
- return from(bytesInput.toByteArray());
- }
-
- /**
- * writes the bytes into a stream
- * @param out
- * @throws IOException
- */
- abstract public void writeAllTo(OutputStream out) throws IOException;
-
- /**
- *
- * @return a new byte array materializing the contents of this input
- * @throws IOException
- */
- public byte[] toByteArray() throws IOException {
- BAOS baos = new BAOS((int)size());
- this.writeAllTo(baos);
- if (DEBUG) LOG.debug("converted " + size() + " to byteArray of " + baos.size() + " bytes");
- return baos.getBuf();
- }
-
- /**
- *
- * @return the size in bytes that would be written
- */
- abstract public long size();
-
- private static final class BAOS extends ByteArrayOutputStream {
- private BAOS(int size) {
- super(size);
- }
-
- public byte[] getBuf() {
- return this.buf;
- }
- }
-
- private static class StreamBytesInput extends BytesInput {
- private static final Log LOG = Log.getLog(BytesInput.StreamBytesInput.class);
- private final InputStream in;
- private final int byteCount;
-
- private StreamBytesInput(InputStream in, int byteCount) {
- super();
- this.in = in;
- this.byteCount = byteCount;
- }
-
- @Override
- public void writeAllTo(OutputStream out) throws IOException {
- if (DEBUG) LOG.debug("write All "+ byteCount + " bytes");
- // TODO: more efficient
- out.write(this.toByteArray());
- }
-
- public byte[] toByteArray() throws IOException {
- if (DEBUG) LOG.debug("read all "+ byteCount + " bytes");
- byte[] buf = new byte[byteCount];
- new DataInputStream(in).readFully(buf);
- return buf;
- }
-
- @Override
- public long size() {
- return byteCount;
- }
-
- }
-
- private static class SequenceBytesIn extends BytesInput {
- private static final Log LOG = Log.getLog(BytesInput.SequenceBytesIn.class);
-
- private final List<BytesInput> inputs;
- private final long size;
-
- private SequenceBytesIn(List<BytesInput> inputs) {
- this.inputs = inputs;
- long total = 0;
- for (BytesInput input : inputs) {
- total += input.size();
- }
- this.size = total;
- }
-
- @SuppressWarnings("unused")
- @Override
- public void writeAllTo(OutputStream out) throws IOException {
- for (BytesInput input : inputs) {
- if (DEBUG) LOG.debug("write " + input.size() + " bytes to out");
- if (DEBUG && input instanceof SequenceBytesIn) LOG.debug("{");
- input.writeAllTo(out);
- if (DEBUG && input instanceof SequenceBytesIn) LOG.debug("}");
- }
- }
-
- @Override
- public long size() {
- return size;
- }
-
- }
-
- private static class IntBytesInput extends BytesInput {
-
- private final int intValue;
-
- public IntBytesInput(int intValue) {
- this.intValue = intValue;
- }
-
- @Override
- public void writeAllTo(OutputStream out) throws IOException {
- BytesUtils.writeIntLittleEndian(out, intValue);
- }
-
- @Override
- public long size() {
- return 4;
- }
-
- }
-
- private static class UnsignedVarIntBytesInput extends BytesInput {
-
- private final int intValue;
-
- public UnsignedVarIntBytesInput(int intValue) {
- this.intValue = intValue;
- }
-
- @Override
- public void writeAllTo(OutputStream out) throws IOException {
- BytesUtils.writeUnsignedVarInt(intValue, out);
- }
-
- @Override
- public long size() {
- int s = 5 - ((Integer.numberOfLeadingZeros(intValue) + 3) / 7);
- return s == 0 ? 1 : s;
- }
- }
-
- private static class EmptyBytesInput extends BytesInput {
-
- @Override
- public void writeAllTo(OutputStream out) throws IOException {
- }
-
- @Override
- public long size() {
- return 0;
- }
-
- }
-
- private static class CapacityBAOSBytesInput extends BytesInput {
-
- private final CapacityByteArrayOutputStream arrayOut;
-
- private CapacityBAOSBytesInput(CapacityByteArrayOutputStream arrayOut) {
- this.arrayOut = arrayOut;
- }
-
- @Override
- public void writeAllTo(OutputStream out) throws IOException {
- arrayOut.writeTo(out);
- }
-
- @Override
- public long size() {
- return arrayOut.size();
- }
-
- }
-
- private static class BAOSBytesInput extends BytesInput {
-
- private final ByteArrayOutputStream arrayOut;
-
- private BAOSBytesInput(ByteArrayOutputStream arrayOut) {
- this.arrayOut = arrayOut;
- }
-
- @Override
- public void writeAllTo(OutputStream out) throws IOException {
- arrayOut.writeTo(out);
- }
-
- @Override
- public long size() {
- return arrayOut.size();
- }
-
- }
-
- private static class ByteArrayBytesInput extends BytesInput {
-
- private final byte[] in;
- private final int offset;
- private final int length;
-
- private ByteArrayBytesInput(byte[] in, int offset, int length) {
- this.in = in;
- this.offset = offset;
- this.length = length;
- }
-
- @Override
- public void writeAllTo(OutputStream out) throws IOException {
- out.write(in, offset, length);
- }
-
- @Override
- public long size() {
- return length;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/CapacityByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/bytes/CapacityByteArrayOutputStream.java b/parquet-encoding/src/main/java/parquet/bytes/CapacityByteArrayOutputStream.java
deleted file mode 100644
index d307471..0000000
--- a/parquet-encoding/src/main/java/parquet/bytes/CapacityByteArrayOutputStream.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.bytes;
-
-import static java.lang.Math.max;
-import static java.lang.Math.pow;
-import static java.lang.String.format;
-import static java.lang.System.arraycopy;
-import static parquet.Preconditions.checkArgument;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import parquet.Log;
-
-/**
- * Similar to a {@link ByteArrayOutputStream}, but uses a different strategy for growing that does not involve copying.
- * Where ByteArrayOutputStream is backed by a single array that "grows" by copying into a new larger array, this output
- * stream grows by allocating a new array (slab) and adding it to a list of previous arrays.
- *
- * Each new slab is allocated to be the same size as all the previous slabs combined, so these allocations become
- * exponentially less frequent, just like ByteArrayOutputStream, with one difference. This output stream accepts a
- * max capacity hint, which is a hint describing the max amount of data that will be written to this stream. As the
- * total size of this stream nears this max, this stream starts to grow linearly instead of exponentially.
- * So new slabs are allocated to be 1/5th of the max capacity hint,
- * instead of equal to the total previous size of all slabs. This is useful because it prevents allocating roughly
- * twice the needed space when a new slab is added just before the stream is done being used.
- *
- * When reusing this stream it will adjust the initial slab size based on the previous data size, aiming for fewer
- * allocations, with the assumption that a similar amount of data will be written to this stream on re-use.
- * See ({@link CapacityByteArrayOutputStream#reset()}).
- *
- * @author Julien Le Dem
- *
- */
-public class CapacityByteArrayOutputStream extends OutputStream {
- private static final Log LOG = Log.getLog(CapacityByteArrayOutputStream.class);
- private static final byte[] EMPTY_SLAB = new byte[0];
-
- private int initialSlabSize;
- private final int maxCapacityHint;
- private final List<byte[]> slabs = new ArrayList<byte[]>();
-
- private byte[] currentSlab;
- private int currentSlabIndex;
- private int bytesAllocated = 0;
- private int bytesUsed = 0;
-
- /**
- * Return an initial slab size such that a CapacityByteArrayOutputStream constructed with it
- * will end up allocating targetNumSlabs in order to reach targetCapacity. This aims to be
- * a balance between the overhead of creating new slabs and wasting memory by eagerly making
- * initial slabs too big.
- *
- * Note that targetCapacity here need not match maxCapacityHint in the constructor of
- * CapacityByteArrayOutputStream, though often that would make sense.
- *
- * @param minSlabSize no matter what we shouldn't make slabs any smaller than this
- * @param targetCapacity after we've allocated targetNumSlabs how much capacity should we have?
- * @param targetNumSlabs how many slabs should it take to reach targetCapacity?
- */
- public static int initialSlabSizeHeuristic(int minSlabSize, int targetCapacity, int targetNumSlabs) {
- // initialSlabSize = (targetCapacity / (2^targetNumSlabs)) means we double targetNumSlabs times
- // before reaching the targetCapacity
- // eg for page size of 1MB we start at 1024 bytes.
- // we also don't want to start too small, so we also apply a minimum.
- return max(minSlabSize, ((int) (targetCapacity / pow(2, targetNumSlabs))));
- }
-
- /**
- * Construct a CapacityByteArrayOutputStream configured such that its initial slab size is
- * determined by {@link #initialSlabSizeHeuristic}, with targetCapacity == maxCapacityHint
- */
- public static CapacityByteArrayOutputStream withTargetNumSlabs(
- int minSlabSize, int maxCapacityHint, int targetNumSlabs) {
-
- return new CapacityByteArrayOutputStream(
- initialSlabSizeHeuristic(minSlabSize, maxCapacityHint, targetNumSlabs),
- maxCapacityHint);
- }
-
- /**
- * Defaults maxCapacityHint to 1MB
- * @param initialSlabSize
- * @deprecated use {@link CapacityByteArrayOutputStream#CapacityByteArrayOutputStream(int, int)}
- */
- @Deprecated
- public CapacityByteArrayOutputStream(int initialSlabSize) {
- this(initialSlabSize, 1024 * 1024);
- }
-
- /**
- * @param initialSlabSize the size to make the first slab
- * @param maxCapacityHint a hint (not guarantee) of the max amount of data written to this stream
- */
- public CapacityByteArrayOutputStream(int initialSlabSize, int maxCapacityHint) {
- checkArgument(initialSlabSize > 0, "initialSlabSize must be > 0");
- checkArgument(maxCapacityHint > 0, "maxCapacityHint must be > 0");
- checkArgument(maxCapacityHint >= initialSlabSize, String.format("maxCapacityHint can't be less than initialSlabSize %d %d", initialSlabSize, maxCapacityHint));
- this.initialSlabSize = initialSlabSize;
- this.maxCapacityHint = maxCapacityHint;
- reset();
- }
-
- /**
- * the new slab is guaranteed to be at least minimumSize
- * @param minimumSize the size of the data we want to copy in the new slab
- */
- private void addSlab(int minimumSize) {
- int nextSlabSize;
-
- if (bytesUsed == 0) {
- nextSlabSize = initialSlabSize;
- } else if (bytesUsed > maxCapacityHint / 5) {
- // to avoid an overhead of up to twice the needed size, we get linear when approaching target page size
- nextSlabSize = maxCapacityHint / 5;
- } else {
- // double the size every time
- nextSlabSize = bytesUsed;
- }
-
- if (nextSlabSize < minimumSize) {
- if (Log.DEBUG) LOG.debug(format("slab size %,d too small for value of size %,d. Bumping up slab size", nextSlabSize, minimumSize));
- nextSlabSize = minimumSize;
- }
-
- if (Log.DEBUG) LOG.debug(format("used %d slabs, adding new slab of size %d", slabs.size(), nextSlabSize));
-
- this.currentSlab = new byte[nextSlabSize];
- this.slabs.add(currentSlab);
- this.bytesAllocated += nextSlabSize;
- this.currentSlabIndex = 0;
- }
-
- @Override
- public void write(int b) {
- if (currentSlabIndex == currentSlab.length) {
- addSlab(1);
- }
- currentSlab[currentSlabIndex] = (byte) b;
- currentSlabIndex += 1;
- bytesUsed += 1;
- }
-
- @Override
- public void write(byte b[], int off, int len) {
- if ((off < 0) || (off > b.length) || (len < 0) ||
- ((off + len) - b.length > 0)) {
- throw new IndexOutOfBoundsException(
- String.format("Given byte array of size %d, with requested length(%d) and offset(%d)", b.length, len, off));
- }
- if (currentSlabIndex + len >= currentSlab.length) {
- final int length1 = currentSlab.length - currentSlabIndex;
- arraycopy(b, off, currentSlab, currentSlabIndex, length1);
- final int length2 = len - length1;
- addSlab(length2);
- arraycopy(b, off + length1, currentSlab, currentSlabIndex, length2);
- currentSlabIndex = length2;
- } else {
- arraycopy(b, off, currentSlab, currentSlabIndex, len);
- currentSlabIndex += len;
- }
- bytesUsed += len;
- }
-
- /**
- * Writes the complete contents of this buffer to the specified output stream argument. the output
- * stream's write method <code>out.write(slab, 0, slab.length)</code>) will be called once per slab.
- *
- * @param out the output stream to which to write the data.
- * @exception IOException if an I/O error occurs.
- */
- public void writeTo(OutputStream out) throws IOException {
- for (int i = 0; i < slabs.size() - 1; i++) {
- final byte[] slab = slabs.get(i);
- out.write(slab);
- }
- out.write(currentSlab, 0, currentSlabIndex);
- }
-
- /**
- * @return The total size in bytes of data written to this stream.
- */
- public long size() {
- return bytesUsed;
- }
-
- /**
- *
- * @return The total size in bytes currently allocated for this stream.
- */
- public int getCapacity() {
- return bytesAllocated;
- }
-
- /**
- * When re-using an instance with reset, it will adjust slab size based on previous data size.
- * The intent is to reuse the same instance for the same type of data (for example, the same column).
- * The assumption is that the size in the buffer will be consistent.
- */
- public void reset() {
- // readjust slab size.
- // 7 = 2^3 - 1 so that doubling the initial size 3 times will get to the same size
- this.initialSlabSize = max(bytesUsed / 7, initialSlabSize);
- if (Log.DEBUG) LOG.debug(String.format("initial slab of size %d", initialSlabSize));
- this.slabs.clear();
- this.bytesAllocated = 0;
- this.bytesUsed = 0;
- this.currentSlab = EMPTY_SLAB;
- this.currentSlabIndex = 0;
- }
-
- /**
- * @return the index of the last value written to this stream, which
- * can be passed to {@link #setByte(long, byte)} in order to change it
- */
- public long getCurrentIndex() {
- checkArgument(bytesUsed > 0, "This is an empty stream");
- return bytesUsed - 1;
- }
-
- /**
- * Replace the byte stored at position index in this stream with value
- *
- * @param index which byte to replace
- * @param value the value to replace it with
- */
- public void setByte(long index, byte value) {
- checkArgument(index < bytesUsed, "Index: " + index + " is >= the current size of: " + bytesUsed);
-
- long seen = 0;
- for (int i = 0; i < slabs.size(); i++) {
- byte[] slab = slabs.get(i);
- if (index < seen + slab.length) {
- // ok found index
- slab[(int)(index-seen)] = value;
- break;
- }
- seen += slab.length;
- }
- }
-
- /**
- * @param prefix a prefix to be used for every new line in the string
- * @return a text representation of the memory usage of this structure
- */
- public String memUsageString(String prefix) {
- return format("%s %s %d slabs, %,d bytes", prefix, getClass().getSimpleName(), slabs.size(), getCapacity());
- }
-
- /**
- * @return the total number of allocated slabs
- */
- int getSlabCount() {
- return slabs.size();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/ConcatenatingByteArrayCollector.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/bytes/ConcatenatingByteArrayCollector.java b/parquet-encoding/src/main/java/parquet/bytes/ConcatenatingByteArrayCollector.java
deleted file mode 100644
index 6a8437b..0000000
--- a/parquet-encoding/src/main/java/parquet/bytes/ConcatenatingByteArrayCollector.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.bytes;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import static java.lang.String.format;
-
-public class ConcatenatingByteArrayCollector extends BytesInput {
- private final List<byte[]> slabs = new ArrayList<byte[]>();
- private long size = 0;
-
- public void collect(BytesInput bytesInput) throws IOException {
- byte[] bytes = bytesInput.toByteArray();
- slabs.add(bytes);
- size += bytes.length;
- }
-
- public void reset() {
- size = 0;
- slabs.clear();
- }
-
- @Override
- public void writeAllTo(OutputStream out) throws IOException {
- for (byte[] slab : slabs) {
- out.write(slab);
- }
- }
-
- @Override
- public long size() {
- return size;
- }
-
- /**
- * @param prefix a prefix to be used for every new line in the string
- * @return a text representation of the memory usage of this structure
- */
- public String memUsageString(String prefix) {
- return format("%s %s %d slabs, %,d bytes", prefix, getClass().getSimpleName(), slabs.size(), size);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataInputStream.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataInputStream.java b/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataInputStream.java
deleted file mode 100644
index 6ea9325..0000000
--- a/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataInputStream.java
+++ /dev/null
@@ -1,424 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.bytes;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * Based on DataInputStream but little endian and without the String/char methods
- *
- * @author Julien Le Dem
- *
- */
-public final class LittleEndianDataInputStream extends InputStream {
-
- private final InputStream in;
-
- /**
- * Creates a LittleEndianDataInputStream that uses the specified
- * underlying InputStream.
- *
- * @param in the specified input stream
- */
- public LittleEndianDataInputStream(InputStream in) {
- this.in = in;
- }
-
- /**
- * See the general contract of the <code>readFully</code>
- * method of <code>DataInput</code>.
- * <p>
- * Bytes
- * for this operation are read from the contained
- * input stream.
- *
- * @param b the buffer into which the data is read.
- * @exception EOFException if this input stream reaches the end before
- * reading all the bytes.
- * @exception IOException the stream has been closed and the contained
- * input stream does not support reading after close, or
- * another I/O error occurs.
- * @see java.io.FilterInputStream#in
- */
- public final void readFully(byte b[]) throws IOException {
- readFully(b, 0, b.length);
- }
-
- /**
- * See the general contract of the <code>readFully</code>
- * method of <code>DataInput</code>.
- * <p>
- * Bytes
- * for this operation are read from the contained
- * input stream.
- *
- * @param b the buffer into which the data is read.
- * @param off the start offset of the data.
- * @param len the number of bytes to read.
- * @exception EOFException if this input stream reaches the end before
- * reading all the bytes.
- * @exception IOException the stream has been closed and the contained
- * input stream does not support reading after close, or
- * another I/O error occurs.
- * @see java.io.FilterInputStream#in
- */
- public final void readFully(byte b[], int off, int len) throws IOException {
- if (len < 0)
- throw new IndexOutOfBoundsException();
- int n = 0;
- while (n < len) {
- int count = in.read(b, off + n, len - n);
- if (count < 0)
- throw new EOFException();
- n += count;
- }
- }
-
- /**
- * See the general contract of the <code>skipBytes</code>
- * method of <code>DataInput</code>.
- * <p>
- * Bytes for this operation are read from the contained
- * input stream.
- *
- * @param n the number of bytes to be skipped.
- * @return the actual number of bytes skipped.
- * @exception IOException if the contained input stream does not support
- * seek, or the stream has been closed and
- * the contained input stream does not support
- * reading after close, or another I/O error occurs.
- */
- public final int skipBytes(int n) throws IOException {
- int total = 0;
- int cur = 0;
-
- while ((total<n) && ((cur = (int) in.skip(n-total)) > 0)) {
- total += cur;
- }
-
- return total;
- }
-
- /**
- * @return
- * @throws IOException
- * @see java.io.InputStream#read()
- */
- public int read() throws IOException {
- return in.read();
- }
-
- /**
- * @return
- * @see java.lang.Object#hashCode()
- */
- public int hashCode() {
- return in.hashCode();
- }
-
- /**
- * @param b
- * @return
- * @throws IOException
- * @see java.io.InputStream#read(byte[])
- */
- public int read(byte[] b) throws IOException {
- return in.read(b);
- }
-
- /**
- * @param obj
- * @return
- * @see java.lang.Object#equals(java.lang.Object)
- */
- public boolean equals(Object obj) {
- return in.equals(obj);
- }
-
- /**
- * @param b
- * @param off
- * @param len
- * @return
- * @throws IOException
- * @see java.io.InputStream#read(byte[], int, int)
- */
- public int read(byte[] b, int off, int len) throws IOException {
- return in.read(b, off, len);
- }
-
- /**
- * @param n
- * @return
- * @throws IOException
- * @see java.io.InputStream#skip(long)
- */
- public long skip(long n) throws IOException {
- return in.skip(n);
- }
-
- /**
- * @return
- * @throws IOException
- * @see java.io.InputStream#available()
- */
- public int available() throws IOException {
- return in.available();
- }
-
- /**
- * @throws IOException
- * @see java.io.InputStream#close()
- */
- public void close() throws IOException {
- in.close();
- }
-
- /**
- * @param readlimit
- * @see java.io.InputStream#mark(int)
- */
- public void mark(int readlimit) {
- in.mark(readlimit);
- }
-
- /**
- * @throws IOException
- * @see java.io.InputStream#reset()
- */
- public void reset() throws IOException {
- in.reset();
- }
-
- /**
- * @return
- * @see java.io.InputStream#markSupported()
- */
- public boolean markSupported() {
- return in.markSupported();
- }
-
- /**
- * See the general contract of the <code>readBoolean</code>
- * method of <code>DataInput</code>.
- * <p>
- * Bytes for this operation are read from the contained
- * input stream.
- *
- * @return the <code>boolean</code> value read.
- * @exception EOFException if this input stream has reached the end.
- * @exception IOException the stream has been closed and the contained
- * input stream does not support reading after close, or
- * another I/O error occurs.
- * @see java.io.FilterInputStream#in
- */
- public final boolean readBoolean() throws IOException {
- int ch = in.read();
- if (ch < 0)
- throw new EOFException();
- return (ch != 0);
- }
-
- /**
- * See the general contract of the <code>readByte</code>
- * method of <code>DataInput</code>.
- * <p>
- * Bytes
- * for this operation are read from the contained
- * input stream.
- *
- * @return the next byte of this input stream as a signed 8-bit
- * <code>byte</code>.
- * @exception EOFException if this input stream has reached the end.
- * @exception IOException the stream has been closed and the contained
- * input stream does not support reading after close, or
- * another I/O error occurs.
- * @see java.io.FilterInputStream#in
- */
- public final byte readByte() throws IOException {
- int ch = in.read();
- if (ch < 0)
- throw new EOFException();
- return (byte)(ch);
- }
-
- /**
- * See the general contract of the <code>readUnsignedByte</code>
- * method of <code>DataInput</code>.
- * <p>
- * Bytes
- * for this operation are read from the contained
- * input stream.
- *
- * @return the next byte of this input stream, interpreted as an
- * unsigned 8-bit number.
- * @exception EOFException if this input stream has reached the end.
- * @exception IOException the stream has been closed and the contained
- * input stream does not support reading after close, or
- * another I/O error occurs.
- * @see java.io.FilterInputStream#in
- */
- public final int readUnsignedByte() throws IOException {
- int ch = in.read();
- if (ch < 0)
- throw new EOFException();
- return ch;
- }
-
- /**
- * Bytes
- * for this operation are read from the contained
- * input stream.
- *
- * @return the next two bytes of this input stream, interpreted as a
- * signed 16-bit number.
- * @exception EOFException if this input stream reaches the end before
- * reading two bytes.
- * @exception IOException the stream has been closed and the contained
- * input stream does not support reading after close, or
- * another I/O error occurs.
- * @see java.io.FilterInputStream#in
- */
- public final short readShort() throws IOException {
- int ch2 = in.read();
- int ch1 = in.read();
- if ((ch1 | ch2) < 0)
- throw new EOFException();
- return (short)((ch1 << 8) + (ch2 << 0));
- }
-
- /**
- * Bytes
- * for this operation are read from the contained
- * input stream.
- *
- * @return the next two bytes of this input stream, interpreted as an
- * unsigned 16-bit integer.
- * @exception EOFException if this input stream reaches the end before
- * reading two bytes.
- * @exception IOException the stream has been closed and the contained
- * input stream does not support reading after close, or
- * another I/O error occurs.
- * @see java.io.FilterInputStream#in
- */
- public final int readUnsignedShort() throws IOException {
- int ch2 = in.read();
- int ch1 = in.read();
- if ((ch1 | ch2) < 0)
- throw new EOFException();
- return (ch1 << 8) + (ch2 << 0);
- }
-
- /**
- * Bytes
- * for this operation are read from the contained
- * input stream.
- *
- * @return the next four bytes of this input stream, interpreted as an
- * <code>int</code>.
- * @exception EOFException if this input stream reaches the end before
- * reading four bytes.
- * @exception IOException the stream has been closed and the contained
- * input stream does not support reading after close, or
- * another I/O error occurs.
- * @see java.io.FilterInputStream#in
- */
- public final int readInt() throws IOException {
- // TODO: has this been benchmarked against two alternate implementations?
- // 1) Integer.reverseBytes(in.readInt())
- // 2) keep a member byte[4], wrapped by an IntBuffer with appropriate endianness set,
- // and call IntBuffer.get()
- // Both seem like they might be faster.
- int ch4 = in.read();
- int ch3 = in.read();
- int ch2 = in.read();
- int ch1 = in.read();
- if ((ch1 | ch2 | ch3 | ch4) < 0)
- throw new EOFException();
- return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4 << 0));
- }
-
- private byte readBuffer[] = new byte[8];
-
- /**
- * Bytes
- * for this operation are read from the contained
- * input stream.
- *
- * @return the next eight bytes of this input stream, interpreted as a
- * <code>long</code>.
- * @exception EOFException if this input stream reaches the end before
- * reading eight bytes.
- * @exception IOException the stream has been closed and the contained
- * input stream does not support reading after close, or
- * another I/O error occurs.
- * @see java.io.FilterInputStream#in
- */
- public final long readLong() throws IOException {
- // TODO: see perf question above in readInt
- readFully(readBuffer, 0, 8);
- return (((long)readBuffer[7] << 56) +
- ((long)(readBuffer[6] & 255) << 48) +
- ((long)(readBuffer[5] & 255) << 40) +
- ((long)(readBuffer[4] & 255) << 32) +
- ((long)(readBuffer[3] & 255) << 24) +
- ((readBuffer[2] & 255) << 16) +
- ((readBuffer[1] & 255) << 8) +
- ((readBuffer[0] & 255) << 0));
- }
-
- /**
- * Bytes
- * for this operation are read from the contained
- * input stream.
- *
- * @return the next four bytes of this input stream, interpreted as a
- * <code>float</code>.
- * @exception EOFException if this input stream reaches the end before
- * reading four bytes.
- * @exception IOException the stream has been closed and the contained
- * input stream does not support reading after close, or
- * another I/O error occurs.
- * @see java.lang.Float#intBitsToFloat(int)
- */
- public final float readFloat() throws IOException {
- return Float.intBitsToFloat(readInt());
- }
-
- /**
- * Bytes
- * for this operation are read from the contained
- * input stream.
- *
- * @return the next eight bytes of this input stream, interpreted as a
- * <code>double</code>.
- * @exception EOFException if this input stream reaches the end before
- * reading eight bytes.
- * @exception IOException the stream has been closed and the contained
- * input stream does not support reading after close, or
- * another I/O error occurs.
- * @see java.lang.Double#longBitsToDouble(long)
- */
- public final double readDouble() throws IOException {
- return Double.longBitsToDouble(readLong());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataOutputStream.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataOutputStream.java b/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataOutputStream.java
deleted file mode 100644
index 2be7299..0000000
--- a/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataOutputStream.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.bytes;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * Based on DataOutputStream but in little endian and without the String/char methods
- *
- * @author Julien Le Dem
- *
- */
-public class LittleEndianDataOutputStream extends OutputStream {
-
- private final OutputStream out;
-
- /**
- * Creates a new data output stream to write data to the specified
- * underlying output stream. The counter <code>written</code> is
- * set to zero.
- *
- * @param out the underlying output stream, to be saved for later
- * use.
- * @see java.io.FilterOutputStream#out
- */
- public LittleEndianDataOutputStream(OutputStream out) {
- this.out = out;
- }
-
- /**
- * Writes the specified byte (the low eight bits of the argument
- * <code>b</code>) to the underlying output stream. If no exception
- * is thrown, the counter <code>written</code> is incremented by
- * <code>1</code>.
- * <p>
- * Implements the <code>write</code> method of <code>OutputStream</code>.
- *
- * @param b the <code>byte</code> to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- public void write(int b) throws IOException {
- out.write(b);
- }
-
- /**
- * Writes <code>len</code> bytes from the specified byte array
- * starting at offset <code>off</code> to the underlying output stream.
- * If no exception is thrown, the counter <code>written</code> is
- * incremented by <code>len</code>.
- *
- * @param b the data.
- * @param off the start offset in the data.
- * @param len the number of bytes to write.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- public void write(byte b[], int off, int len) throws IOException {
- out.write(b, off, len);
- }
-
- /**
- * Flushes this data output stream. This forces any buffered output
- * bytes to be written out to the stream.
- * <p>
- * The <code>flush</code> method of <code>DataOutputStream</code>
- * calls the <code>flush</code> method of its underlying output stream.
- *
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- * @see java.io.OutputStream#flush()
- */
- public void flush() throws IOException {
- out.flush();
- }
-
- /**
- * Writes a <code>boolean</code> to the underlying output stream as
- * a 1-byte value. The value <code>true</code> is written out as the
- * value <code>(byte)1</code>; the value <code>false</code> is
- * written out as the value <code>(byte)0</code>. If no exception is
- * thrown, the counter <code>written</code> is incremented by
- * <code>1</code>.
- *
- * @param v a <code>boolean</code> value to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- public final void writeBoolean(boolean v) throws IOException {
- out.write(v ? 1 : 0);
- }
-
- /**
- * Writes out a <code>byte</code> to the underlying output stream as
- * a 1-byte value. If no exception is thrown, the counter
- * <code>written</code> is incremented by <code>1</code>.
- *
- * @param v a <code>byte</code> value to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- public final void writeByte(int v) throws IOException {
- out.write(v);
- }
-
- /**
- * Writes a <code>short</code> to the underlying output stream as two
- * bytes, low byte first. If no exception is thrown, the counter
- * <code>written</code> is incremented by <code>2</code>.
- *
- * @param v a <code>short</code> to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- public final void writeShort(int v) throws IOException {
- out.write((v >>> 0) & 0xFF);
- out.write((v >>> 8) & 0xFF);
- }
-
- /**
- * Writes an <code>int</code> to the underlying output stream as four
- * bytes, low byte first. If no exception is thrown, the counter
- * <code>written</code> is incremented by <code>4</code>.
- *
- * @param v an <code>int</code> to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- public final void writeInt(int v) throws IOException {
- // TODO: see note in LittleEndianDataInputStream: maybe faster
- // to use Integer.reverseBytes() and then writeInt, or a ByteBuffer
- // approach
- out.write((v >>> 0) & 0xFF);
- out.write((v >>> 8) & 0xFF);
- out.write((v >>> 16) & 0xFF);
- out.write((v >>> 24) & 0xFF);
- }
-
- private byte writeBuffer[] = new byte[8];
-
- /**
- * Writes a <code>long</code> to the underlying output stream as eight
- * bytes, low byte first. In no exception is thrown, the counter
- * <code>written</code> is incremented by <code>8</code>.
- *
- * @param v a <code>long</code> to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- */
- public final void writeLong(long v) throws IOException {
- writeBuffer[7] = (byte)(v >>> 56);
- writeBuffer[6] = (byte)(v >>> 48);
- writeBuffer[5] = (byte)(v >>> 40);
- writeBuffer[4] = (byte)(v >>> 32);
- writeBuffer[3] = (byte)(v >>> 24);
- writeBuffer[2] = (byte)(v >>> 16);
- writeBuffer[1] = (byte)(v >>> 8);
- writeBuffer[0] = (byte)(v >>> 0);
- out.write(writeBuffer, 0, 8);
- }
-
- /**
- * Converts the float argument to an <code>int</code> using the
- * <code>floatToIntBits</code> method in class <code>Float</code>,
- * and then writes that <code>int</code> value to the underlying
- * output stream as a 4-byte quantity, low byte first. If no
- * exception is thrown, the counter <code>written</code> is
- * incremented by <code>4</code>.
- *
- * @param v a <code>float</code> value to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- * @see java.lang.Float#floatToIntBits(float)
- */
- public final void writeFloat(float v) throws IOException {
- writeInt(Float.floatToIntBits(v));
- }
-
- /**
- * Converts the double argument to a <code>long</code> using the
- * <code>doubleToLongBits</code> method in class <code>Double</code>,
- * and then writes that <code>long</code> value to the underlying
- * output stream as an 8-byte quantity, low byte first. If no
- * exception is thrown, the counter <code>written</code> is
- * incremented by <code>8</code>.
- *
- * @param v a <code>double</code> value to be written.
- * @exception IOException if an I/O error occurs.
- * @see java.io.FilterOutputStream#out
- * @see java.lang.Double#doubleToLongBits(double)
- */
- public final void writeDouble(double v) throws IOException {
- writeLong(Double.doubleToLongBits(v));
- }
-
-}