You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by bl...@apache.org on 2015/04/28 01:12:13 UTC
[16/51] [partial] parquet-mr git commit: PARQUET-23: Rename to
org.apache.parquet.
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestByteBitPacking.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestByteBitPacking.java b/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestByteBitPacking.java
deleted file mode 100644
index 6c1ffb7..0000000
--- a/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestByteBitPacking.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.column.values.bitpacking;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import parquet.Log;
-import parquet.column.values.bitpacking.BitPacking.BitPackingReader;
-import parquet.column.values.bitpacking.BitPacking.BitPackingWriter;
-
-public class TestByteBitPacking {
- private static final Log LOG = Log.getLog(TestByteBitPacking.class);
-
- @Test
- public void testPackUnPack() {
- LOG.debug("");
- LOG.debug("testPackUnPack");
- for (int i = 1; i < 32; i++) {
- LOG.debug("Width: " + i);
- int[] unpacked = new int[32];
- int[] values = generateValues(i);
- packUnpack(Packer.BIG_ENDIAN.newBytePacker(i), values, unpacked);
- LOG.debug("Output: " + TestBitPacking.toString(unpacked));
- Assert.assertArrayEquals("width "+i, values, unpacked);
- }
- }
-
- private void packUnpack(BytePacker packer, int[] values, int[] unpacked) {
- byte[] packed = new byte[packer.getBitWidth() * 4];
- packer.pack32Values(values, 0, packed, 0);
- LOG.debug("packed: " + TestBitPacking.toString(packed));
- packer.unpack32Values(packed, 0, unpacked, 0);
- }
-
- private int[] generateValues(int bitWidth) {
- int[] values = new int[32];
- for (int j = 0; j < values.length; j++) {
- values[j] = (int)(Math.random() * 100000) % (int)Math.pow(2, bitWidth);
- }
- LOG.debug("Input: " + TestBitPacking.toString(values));
- return values;
- }
-
- @Test
- public void testPackUnPackAgainstHandWritten() throws IOException {
- LOG.debug("");
- LOG.debug("testPackUnPackAgainstHandWritten");
- for (int i = 1; i < 8; i++) {
- LOG.debug("Width: " + i);
- byte[] packed = new byte[i * 4];
- int[] unpacked = new int[32];
- int[] values = generateValues(i);
-
- // pack generated
- final BytePacker packer = Packer.BIG_ENDIAN.newBytePacker(i);
- packer.pack32Values(values, 0, packed, 0);
-
- LOG.debug("Generated: " + TestBitPacking.toString(packed));
-
- // pack manual
- final ByteArrayOutputStream manualOut = new ByteArrayOutputStream();
- final BitPackingWriter writer = BitPacking.getBitPackingWriter(i, manualOut);
- for (int j = 0; j < values.length; j++) {
- writer.write(values[j]);
- }
- final byte[] packedManualAsBytes = manualOut.toByteArray();
- LOG.debug("Manual: " + TestBitPacking.toString(packedManualAsBytes));
-
- // unpack manual
- final BitPackingReader reader = BitPacking.createBitPackingReader(i, new ByteArrayInputStream(packed), 32);
- for (int j = 0; j < unpacked.length; j++) {
- unpacked[j] = reader.read();
- }
-
- LOG.debug("Output: " + TestBitPacking.toString(unpacked));
- Assert.assertArrayEquals("width " + i, values, unpacked);
- }
- }
-
- @Test
- public void testPackUnPackAgainstLemire() throws IOException {
- for (Packer pack: Packer.values()) {
- LOG.debug("");
- LOG.debug("testPackUnPackAgainstLemire " + pack.name());
- for (int i = 1; i < 32; i++) {
- LOG.debug("Width: " + i);
- int[] packed = new int[i];
- int[] unpacked = new int[32];
- int[] values = generateValues(i);
-
- // pack lemire
- final IntPacker packer = pack.newIntPacker(i);
- packer.pack32Values(values, 0, packed, 0);
- // convert to bytes
- final ByteArrayOutputStream lemireOut = new ByteArrayOutputStream();
- for (int v : packed) {
- switch(pack) {
- case LITTLE_ENDIAN:
- lemireOut.write((v >>> 0) & 0xFF);
- lemireOut.write((v >>> 8) & 0xFF);
- lemireOut.write((v >>> 16) & 0xFF);
- lemireOut.write((v >>> 24) & 0xFF);
- break;
- case BIG_ENDIAN:
- lemireOut.write((v >>> 24) & 0xFF);
- lemireOut.write((v >>> 16) & 0xFF);
- lemireOut.write((v >>> 8) & 0xFF);
- lemireOut.write((v >>> 0) & 0xFF);
- break;
- }
- }
- final byte[] packedByLemireAsBytes = lemireOut.toByteArray();
- LOG.debug("Lemire out: " + TestBitPacking.toString(packedByLemireAsBytes));
-
- // pack manual
- final BytePacker bytePacker = pack.newBytePacker(i);
- byte[] packedGenerated = new byte[i * 4];
- bytePacker.pack32Values(values, 0, packedGenerated, 0);
- LOG.debug("Gener. out: " + TestBitPacking.toString(packedGenerated));
- Assert.assertEquals(pack.name() + " width " + i, TestBitPacking.toString(packedByLemireAsBytes), TestBitPacking.toString(packedGenerated));
-
- bytePacker.unpack32Values(packedByLemireAsBytes, 0, unpacked, 0);
- LOG.debug("Output: " + TestBitPacking.toString(unpacked));
-
- Assert.assertArrayEquals("width " + i, values, unpacked);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestLemireBitPacking.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestLemireBitPacking.java b/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestLemireBitPacking.java
deleted file mode 100644
index 0929865..0000000
--- a/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestLemireBitPacking.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.column.values.bitpacking;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import parquet.Log;
-import parquet.column.values.bitpacking.BitPacking.BitPackingReader;
-import parquet.column.values.bitpacking.BitPacking.BitPackingWriter;
-
-public class TestLemireBitPacking {
- private static final Log LOG = Log.getLog(TestLemireBitPacking.class);
-
- @Test
- public void testPackUnPack() {
- for (Packer packer : Packer.values()) {
- LOG.debug("");
- LOG.debug("testPackUnPack");
- for (int i = 1; i < 32; i++) {
- LOG.debug("Width: " + i);
- int[] values = generateValues(i);
- int[] unpacked = new int[32];
- {
- packUnpack(packer.newIntPacker(i), values, unpacked);
- LOG.debug("int based Output " + packer.name() + ": " + TestBitPacking.toString(unpacked));
- Assert.assertArrayEquals(packer.name() + " width "+i, values, unpacked);
- }
- {
- packUnpack(packer.newBytePacker(i), values, unpacked);
- LOG.debug("byte based Output " + packer.name() + ": " + TestBitPacking.toString(unpacked));
- Assert.assertArrayEquals(packer.name() + " width "+i, values, unpacked);
- }
- }
- }
- }
-
- private void packUnpack(IntPacker packer, int[] values, int[] unpacked) {
- int[] packed = new int[packer.getBitWidth()];
- packer.pack32Values(values, 0, packed, 0);
- packer.unpack32Values(packed, 0, unpacked, 0);
- }
-
- private void packUnpack(BytePacker packer, int[] values, int[] unpacked) {
- byte[] packed = new byte[packer.getBitWidth() * 4];
- packer.pack32Values(values, 0, packed, 0);
- packer.unpack32Values(packed, 0, unpacked, 0);
- }
-
- private int[] generateValues(int bitWidth) {
- int[] values = new int[32];
- for (int j = 0; j < values.length; j++) {
- values[j] = (int)(Math.random() * 100000) % (int)Math.pow(2, bitWidth);
- }
- LOG.debug("Input: " + TestBitPacking.toString(values));
- return values;
- }
-
- @Test
- public void testPackUnPackAgainstHandWritten() throws IOException {
- LOG.debug("");
- LOG.debug("testPackUnPackAgainstHandWritten");
- for (int i = 1; i < 8; i++) {
- LOG.debug("Width: " + i);
- int[] packed = new int[i];
- int[] unpacked = new int[32];
- int[] values = generateValues(i);
-
- // pack lemire
- final IntPacker packer = Packer.BIG_ENDIAN.newIntPacker(i);
- packer.pack32Values(values, 0, packed, 0);
- // convert to ints
- final ByteArrayOutputStream lemireOut = new ByteArrayOutputStream();
- for (int v : packed) {
- lemireOut.write((v >>> 24) & 0xFF);
- lemireOut.write((v >>> 16) & 0xFF);
- lemireOut.write((v >>> 8) & 0xFF);
- lemireOut.write((v >>> 0) & 0xFF);
- }
- final byte[] packedByLemireAsBytes = lemireOut.toByteArray();
- LOG.debug("Lemire: " + TestBitPacking.toString(packedByLemireAsBytes));
-
- // pack manual
- final ByteArrayOutputStream manualOut = new ByteArrayOutputStream();
- final BitPackingWriter writer = BitPacking.getBitPackingWriter(i, manualOut);
- for (int j = 0; j < values.length; j++) {
- writer.write(values[j]);
- }
- final byte[] packedManualAsBytes = manualOut.toByteArray();
- LOG.debug("Manual: " + TestBitPacking.toString(packedManualAsBytes));
-
- // unpack manual
- final BitPackingReader reader = BitPacking.createBitPackingReader(i, new ByteArrayInputStream(packedByLemireAsBytes), 32);
- for (int j = 0; j < unpacked.length; j++) {
- unpacked[j] = reader.read();
- }
-
- LOG.debug("Output: " + TestBitPacking.toString(unpacked));
- Assert.assertArrayEquals("width " + i, values, unpacked);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-generator/pom.xml b/parquet-generator/pom.xml
index 1931d59..52c0cad 100644
--- a/parquet-generator/pom.xml
+++ b/parquet-generator/pom.xml
@@ -18,7 +18,7 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
- <groupId>com.twitter</groupId>
+ <groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
<version>1.7.0-incubating-SNAPSHOT</version>
@@ -37,7 +37,7 @@
<dependencies>
<dependency>
- <groupId>com.twitter</groupId>
+ <groupId>org.apache.parquet</groupId>
<artifactId>parquet-common</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/org/apache/parquet/encoding/Generator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/org/apache/parquet/encoding/Generator.java b/parquet-generator/src/main/java/org/apache/parquet/encoding/Generator.java
new file mode 100644
index 0000000..641e76d
--- /dev/null
+++ b/parquet-generator/src/main/java/org/apache/parquet/encoding/Generator.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.encoding;
+
+import org.apache.parquet.encoding.bitpacking.ByteBasedBitPackingGenerator;
+import org.apache.parquet.encoding.bitpacking.IntBasedBitPackingGenerator;
+
+/**
+ * main class for code generation hook in build for encodings generation
+ *
+ * @author Julien Le Dem
+ *
+ */
+public class Generator {
+
+ public static void main(String[] args) throws Exception {
+ IntBasedBitPackingGenerator.main(args);
+ ByteBasedBitPackingGenerator.main(args);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java b/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
new file mode 100644
index 0000000..2249cbc
--- /dev/null
+++ b/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.encoding.bitpacking;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+import org.apache.parquet.bytes.BytesUtils;
+
+/**
+ *
+ * This class generates bit packers that pack the most significant bit first.
+ * The result of the generation is checked in. To regenerate the code run this class and check in the result.
+ *
+ * TODO: remove the unnecessary masks for perf
+ *
+ * @author Julien Le Dem
+ *
+ */
+public class ByteBasedBitPackingGenerator {
+
+ private static final String CLASS_NAME_PREFIX = "ByteBitPacking";
+ private static final int PACKER_COUNT = 32;
+
+ public static void main(String[] args) throws Exception {
+ String basePath = args[0];
+ generateScheme(CLASS_NAME_PREFIX + "BE", true, basePath);
+ generateScheme(CLASS_NAME_PREFIX + "LE", false, basePath);
+ }
+
+ private static void generateScheme(String className, boolean msbFirst, String basePath) throws IOException {
+ final File file = new File(basePath + "/org/apache/parquet/column/values/bitpacking/" + className + ".java").getAbsoluteFile();
+ if (!file.getParentFile().exists()) {
+ file.getParentFile().mkdirs();
+ }
+ FileWriter fw = new FileWriter(file);
+ fw.append("package org.apache.parquet.column.values.bitpacking;\n");
+ fw.append("\n");
+ fw.append("/**\n");
+ if (msbFirst) {
+ fw.append(" * Packs from the Most Significant Bit first\n");
+ } else {
+ fw.append(" * Packs from the Least Significant Bit first\n");
+ }
+ fw.append(" * \n");
+ fw.append(" * @author automatically generated\n");
+ fw.append(" * @see ByteBasedBitPackingGenerator\n");
+ fw.append(" *\n");
+ fw.append(" */\n");
+ fw.append("public abstract class " + className + " {\n");
+ fw.append("\n");
+ fw.append(" private static final BytePacker[] packers = new BytePacker[33];\n");
+ fw.append(" static {\n");
+ for (int i = 0; i <= PACKER_COUNT; i++) {
+ fw.append(" packers[" + i + "] = new Packer" + i + "();\n");
+ }
+ fw.append(" }\n");
+ fw.append("\n");
+ fw.append(" public static final BytePackerFactory factory = new BytePackerFactory() {\n");
+ fw.append(" public BytePacker newBytePacker(int bitWidth) {\n");
+ fw.append(" return packers[bitWidth];\n");
+ fw.append(" }\n");
+ fw.append(" };\n");
+ fw.append("\n");
+ for (int i = 0; i <= PACKER_COUNT; i++) {
+ generateClass(fw, i, msbFirst);
+ fw.append("\n");
+ }
+ fw.append("}\n");
+ fw.close();
+ }
+
+ private static void generateClass(FileWriter fw, int bitWidth, boolean msbFirst) throws IOException {
+ fw.append(" private static final class Packer" + bitWidth + " extends BytePacker {\n");
+ fw.append("\n");
+ fw.append(" private Packer" + bitWidth + "() {\n");
+ fw.append(" super("+bitWidth+");\n");
+ fw.append(" }\n");
+ fw.append("\n");
+ // Packing
+ generatePack(fw, bitWidth, 1, msbFirst);
+ generatePack(fw, bitWidth, 4, msbFirst);
+
+ // Unpacking
+ generateUnpack(fw, bitWidth, 1, msbFirst);
+ generateUnpack(fw, bitWidth, 4, msbFirst);
+
+ fw.append(" }\n");
+ }
+
+ private static int getShift(FileWriter fw, int bitWidth, boolean msbFirst,
+ int byteIndex, int valueIndex) throws IOException {
+ // relative positions of the start and end of the value to the start and end of the byte
+ int valueStartBitIndex = (valueIndex * bitWidth) - (8 * (byteIndex));
+ int valueEndBitIndex = ((valueIndex + 1) * bitWidth) - (8 * (byteIndex + 1));
+
+ // boundaries of the current value that we want
+ int valueStartBitWanted;
+ int valueEndBitWanted;
+ // boundaries of the current byte that will receive them
+ int byteStartBitWanted;
+ int byteEndBitWanted;
+
+ int shift;
+
+ if (msbFirst) {
+ valueStartBitWanted = valueStartBitIndex < 0 ? bitWidth - 1 + valueStartBitIndex : bitWidth - 1;
+ valueEndBitWanted = valueEndBitIndex > 0 ? valueEndBitIndex : 0;
+ byteStartBitWanted = valueStartBitIndex < 0 ? 8 : 7 - valueStartBitIndex;
+ byteEndBitWanted = valueEndBitIndex > 0 ? 0 : -valueEndBitIndex;
+ shift = valueEndBitWanted - byteEndBitWanted;
+ } else {
+ valueStartBitWanted = bitWidth - 1 - (valueEndBitIndex > 0 ? valueEndBitIndex : 0);
+ valueEndBitWanted = bitWidth - 1 - (valueStartBitIndex < 0 ? bitWidth - 1 + valueStartBitIndex : bitWidth - 1);
+ byteStartBitWanted = 7 - (valueEndBitIndex > 0 ? 0 : -valueEndBitIndex);
+ byteEndBitWanted = 7 - (valueStartBitIndex < 0 ? 8 : 7 - valueStartBitIndex);
+ shift = valueStartBitWanted - byteStartBitWanted;
+ }
+
+ visualizeAlignment(
+ fw, bitWidth, valueEndBitIndex,
+ valueStartBitWanted, valueEndBitWanted,
+ byteStartBitWanted, byteEndBitWanted,
+ shift
+ );
+ return shift;
+ }
+
+ private static void visualizeAlignment(FileWriter fw, int bitWidth,
+ int valueEndBitIndex, int valueStartBitWanted, int valueEndBitWanted,
+ int byteStartBitWanted, int byteEndBitWanted, int shift) throws IOException {
+ // ASCII art to visualize what is happening
+ fw.append("//");
+ int buf = 2 + Math.max(0, bitWidth + 8);
+ for (int i = 0; i < buf; i++) {
+ fw.append(" ");
+ }
+ fw.append("[");
+ for (int i = 7; i >= 0; i--) {
+ if (i<=byteStartBitWanted && i>=byteEndBitWanted) {
+ fw.append(String.valueOf(i));
+ } else {
+ fw.append("_");
+ }
+ }
+ fw.append("]\n //");
+ for (int i = 0; i < buf + (8 - bitWidth + shift); i++) {
+ fw.append(" ");
+ }
+ fw.append("[");
+ for (int i = bitWidth - 1; i >= 0 ; i--) {
+ if (i<=valueStartBitWanted && i>=valueEndBitWanted) {
+ fw.append(String.valueOf(i % 10));
+ } else {
+ fw.append("_");
+ }
+ }
+ fw.append("]\n");
+ fw.append(" ");
+ }
+
+ private static void generatePack(FileWriter fw, int bitWidth, int batch, boolean msbFirst) throws IOException {
+ int mask = genMask(bitWidth);
+ fw.append(" public final void pack" + (batch * 8) + "Values(final int[] in, final int inPos, final byte[] out, final int outPos) {\n");
+ for (int byteIndex = 0; byteIndex < bitWidth * batch; ++byteIndex) {
+ fw.append(" out[" + align(byteIndex, 2) + " + outPos] = (byte)((\n");
+ int startIndex = (byteIndex * 8) / bitWidth;
+ int endIndex = ((byteIndex + 1) * 8 + bitWidth - 1) / bitWidth;
+ for (int valueIndex = startIndex; valueIndex < endIndex; valueIndex++) {
+
+ if (valueIndex == startIndex) {
+ fw.append(" ");
+ } else {
+ fw.append("\n | ");
+ }
+ int shift = getShift(fw, bitWidth, msbFirst, byteIndex, valueIndex);
+
+ String shiftString = ""; // used when shift == 0
+ if (shift > 0) {
+ shiftString = " >>> " + shift;
+ } else if (shift < 0) {
+ shiftString = " << " + ( - shift);
+ }
+ fw.append("((in[" + align(valueIndex, 2) + " + inPos] & " + mask + ")" + shiftString + ")");
+ }
+ fw.append(") & 255);\n");
+ }
+ fw.append(" }\n");
+ }
+
+ private static void generateUnpack(FileWriter fw, int bitWidth, int batch, boolean msbFirst)
+ throws IOException {
+ fw.append(" public final void unpack" + (batch * 8) + "Values(final byte[] in, final int inPos, final int[] out, final int outPos) {\n");
+ if (bitWidth > 0) {
+ int mask = genMask(bitWidth);
+ for (int valueIndex = 0; valueIndex < (batch * 8); ++valueIndex) {
+ fw.append(" out[" + align(valueIndex, 2) + " + outPos] =\n");
+
+ int startIndex = valueIndex * bitWidth / 8;
+ int endIndex = BytesUtils.paddedByteCountFromBits((valueIndex + 1) * bitWidth);
+
+ for (int byteIndex = startIndex; byteIndex < endIndex; byteIndex++) {
+ if (byteIndex == startIndex) {
+ fw.append(" ");
+ } else {
+ fw.append("\n | ");
+ }
+ int shift = getShift(fw, bitWidth, msbFirst, byteIndex, valueIndex);
+
+ String shiftString = ""; // when shift == 0
+ if (shift < 0) {
+ shiftString = ">>> " + (-shift);
+ } else if (shift > 0){
+ shiftString = "<< " + shift;
+ }
+ fw.append(" (((((int)in[" + align(byteIndex, 2) + " + inPos]) & 255) " + shiftString + ") & " + mask + ")");
+ }
+ fw.append(";\n");
+ }
+ }
+ fw.append(" }\n");
+ }
+
+ private static int genMask(int bitWidth) {
+ int mask = 0;
+ for (int i = 0; i < bitWidth; i++) {
+ mask <<= 1;
+ mask |= 1;
+ }
+ return mask;
+ }
+
+ private static String align(int value, int digits) {
+ final String valueString = String.valueOf(value);
+ StringBuilder result = new StringBuilder();
+ for (int i = valueString.length(); i < digits; i++) {
+ result.append(" ");
+ }
+ result.append(valueString);
+ return result.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java b/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java
new file mode 100644
index 0000000..e472663
--- /dev/null
+++ b/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.encoding.bitpacking;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+/**
+ * Scheme designed by D. Lemire
+ *
+ * This is a re-implementation of The scheme released under Apache License Version 2.0
+ * at https://github.com/lemire/JavaFastPFOR/blob/master/src/integercompression/BitPacking.java
+ *
+ * It generate two classes:
+ * - LemireBitPackingLE, the original scheme, filling the LSB first
+ * - LemireBitPackingBE, the scheme modified to fill the MSB first (and match our existing bit packing)
+ *
+ * The result of the generation is checked in. To regenerate the code run this class and check in the result.
+ *
+ * The generated classes pack the values into arrays of ints (as opposed to arrays of bytes) based on a given bit width.
+ *
+ * Note: This is not really used for now as the hadoop API does not really let write int[]. We need to revisit this
+ *
+ * @author Julien Le Dem
+ *
+ */
+public class IntBasedBitPackingGenerator {
+
+ private static final String CLASS_NAME_PREFIX = "LemireBitPacking";
+
+ public static void main(String[] args) throws Exception {
+ String basePath = args[0];
+ generateScheme(CLASS_NAME_PREFIX + "BE", true, basePath);
+ generateScheme(CLASS_NAME_PREFIX + "LE", false, basePath);
+ }
+
+ private static void generateScheme(String className, boolean msbFirst, String basePath) throws IOException {
+ final File file = new File(basePath + "/org/apache/parquet/column/values/bitpacking/" + className + ".java").getAbsoluteFile();
+ if (!file.getParentFile().exists()) {
+ file.getParentFile().mkdirs();
+ }
+ FileWriter fw = new FileWriter(file);
+ fw.append("package org.apache.parquet.column.values.bitpacking;\n");
+ fw.append("\n");
+ fw.append("/**\n");
+ fw.append(" * Based on the original implementation at at https://github.com/lemire/JavaFastPFOR/blob/master/src/integercompression/BitPacking.java\n");
+ fw.append(" * Which is released under the\n");
+ fw.append(" * Apache License Version 2.0 http://www.apache.org/licenses/.\n");
+ fw.append(" * By Daniel Lemire, http://lemire.me/en/\n");
+ fw.append(" * \n");
+ fw.append(" * Scheme designed by D. Lemire\n");
+ if (msbFirst) {
+ fw.append(" * Adapted to pack from the Most Significant Bit first\n");
+ }
+ fw.append(" * \n");
+ fw.append(" * @author automatically generated\n");
+ fw.append(" * @see IntBasedBitPackingGenerator\n");
+ fw.append(" *\n");
+ fw.append(" */\n");
+ fw.append("abstract class " + className + " {\n");
+ fw.append("\n");
+ fw.append(" private static final IntPacker[] packers = new IntPacker[32];\n");
+ fw.append(" static {\n");
+ for (int i = 0; i < 32; i++) {
+ fw.append(" packers[" + i + "] = new Packer" + i + "();\n");
+ }
+ fw.append(" }\n");
+ fw.append("\n");
+ fw.append(" public static final IntPackerFactory factory = new IntPackerFactory() {\n");
+ fw.append(" public IntPacker newIntPacker(int bitWidth) {\n");
+ fw.append(" return packers[bitWidth];\n");
+ fw.append(" }\n");
+ fw.append(" };\n");
+ fw.append("\n");
+ for (int i = 0; i < 32; i++) {
+ generateClass(fw, i, msbFirst);
+ fw.append("\n");
+ }
+ fw.append("}\n");
+ fw.close();
+ }
+
+ private static void generateClass(FileWriter fw, int bitWidth, boolean msbFirst) throws IOException {
+ int mask = 0;
+ for (int i = 0; i < bitWidth; i++) {
+ mask <<= 1;
+ mask |= 1;
+ }
+ fw.append(" private static final class Packer" + bitWidth + " extends IntPacker {\n");
+ fw.append("\n");
+ fw.append(" private Packer" + bitWidth + "() {\n");
+ fw.append(" super(" + bitWidth + ");\n");
+ fw.append(" }\n");
+ fw.append("\n");
+ // Packing
+ fw.append(" public final void pack32Values(final int[] in, final int inPos, final int[] out, final int outPos) {\n");
+ for (int i = 0; i < bitWidth; ++i) {
+ fw.append(" out[" + align(i, 2) + " + outPos] =\n");
+ int startIndex = (i * 32) / bitWidth;
+ int endIndex = ((i + 1) * 32 + bitWidth - 1) / bitWidth;
+ for (int j = startIndex; j < endIndex; j++) {
+ if (j == startIndex) {
+ fw.append(" ");
+ } else {
+ fw.append("\n | ");
+ }
+ String shiftString = getPackShiftString(bitWidth, i, startIndex, j, msbFirst);
+ fw.append("((in[" + align(j, 2) + " + inPos] & " + mask + ")" + shiftString + ")");
+ }
+ fw.append(";\n");
+ }
+ fw.append(" }\n");
+
+ // Unpacking
+ fw.append(" public final void unpack32Values(final int[] in, final int inPos, final int[] out, final int outPos) {\n");
+ if (bitWidth > 0) {
+ for (int i = 0; i < 32; ++i) {
+ fw.append(" out[" + align(i, 2) + " + outPos] =");
+ int byteIndex = i * bitWidth / 32;
+ String shiftString = getUnpackShiftString(bitWidth, i, msbFirst);
+ fw.append(" ((in[" + align(byteIndex, 2) + " + inPos] " + shiftString + ") & " + mask + ")");
+ if (((i + 1) * bitWidth - 1 ) / 32 != byteIndex) {
+ // reading the end of the value from next int
+ int bitsRead = ((i + 1) * bitWidth - 1) % 32 + 1;
+ fw.append(" | ((in[" + align(byteIndex + 1, 2) + " + inPos]");
+ if (msbFirst) {
+ fw.append(") >>> " + align(32 - bitsRead, 2) + ")");
+ } else {
+ int lowerMask = 0;
+ for (int j = 0; j < bitsRead; j++) {
+ lowerMask <<= 1;
+ lowerMask |= 1;
+ }
+ fw.append(" & " + lowerMask + ") << " + align(bitWidth - bitsRead, 2) + ")");
+ }
+ }
+ fw.append(";\n");
+ }
+ }
+ fw.append(" }\n");
+ fw.append(" }\n");
+ }
+
+ private static String getUnpackShiftString(int bitWidth, int i, boolean msbFirst) {
+ final int regularShift = i * bitWidth % 32;
+ String shiftString;
+ if (msbFirst) {
+ int shift = 32 - (regularShift + bitWidth);
+ if (shift < 0) {
+ shiftString = "<< " + align(-shift, 2);
+ } else {
+ shiftString = ">>> " + align(shift, 2);
+ }
+ } else {
+ shiftString = ">>> " + align(regularShift, 2);
+ }
+ return shiftString;
+ }
+
+ private static String getPackShiftString(int bitWidth, int integerIndex, int startIndex, int valueIndex, boolean msbFirst) {
+ String shiftString;
+ int regularShift = (valueIndex * bitWidth) % 32;
+ if (msbFirst) { // filling most significant bit first
+ int shift = 32 - (regularShift + bitWidth);
+ if (valueIndex == startIndex && (integerIndex * 32) % bitWidth != 0) {
+ // end of last value from previous int
+ shiftString = " << " + align(32 - (((valueIndex + 1) * bitWidth) % 32), 2);
+ } else if (shift < 0) {
+ // partial last value
+ shiftString = " >>> " + align(-shift, 2);
+ } else {
+ shiftString = " << " + align(shift, 2);
+ }
+ } else { // filling least significant bit first
+ if (valueIndex == startIndex && (integerIndex * 32) % bitWidth != 0) {
+ // end of last value from previous int
+ shiftString = " >>> " + align(32 - regularShift, 2);
+ } else {
+ shiftString = " << " + align(regularShift, 2);
+ }
+ }
+ return shiftString;
+ }
+
+ private static String align(int value, int digits) {
+ final String valueString = String.valueOf(value);
+ StringBuilder result = new StringBuilder();
+ for (int i = valueString.length(); i < digits; i++) {
+ result.append(" ");
+ }
+ result.append(valueString);
+ return result.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/org/apache/parquet/filter2/Generator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/org/apache/parquet/filter2/Generator.java b/parquet-generator/src/main/java/org/apache/parquet/filter2/Generator.java
new file mode 100644
index 0000000..042015c
--- /dev/null
+++ b/parquet-generator/src/main/java/org/apache/parquet/filter2/Generator.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.filter2;
+
+/**
+ * main class for code generation hook in build for filter2 generation
+ */
+public class Generator {
+ public static void main(String[] args) throws Exception {
+ IncrementallyUpdatedFilterPredicateGenerator.main(args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/org/apache/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/org/apache/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java b/parquet-generator/src/main/java/org/apache/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java
new file mode 100644
index 0000000..1dfaf6f
--- /dev/null
+++ b/parquet-generator/src/main/java/org/apache/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.filter2;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+public class IncrementallyUpdatedFilterPredicateGenerator {
+
+ public static void main(String[] args) throws IOException {
+ File srcFile = new File(args[0] + "/org/apache/parquet/filter2/recordlevel/IncrementallyUpdatedFilterPredicateBuilder.java");
+ srcFile = srcFile.getAbsoluteFile();
+ File parent = srcFile.getParentFile();
+ if (!parent.exists()) {
+ if (!parent.mkdirs()) {
+ throw new IOException("Couldn't mkdirs for " + parent);
+ }
+ }
+ new IncrementallyUpdatedFilterPredicateGenerator(srcFile).run();
+ }
+
+ private final FileWriter writer;
+
+ public IncrementallyUpdatedFilterPredicateGenerator(File file) throws IOException {
+ this.writer = new FileWriter(file);
+ }
+
+ private static class TypeInfo {
+ public final String className;
+ public final String primitiveName;
+ public final boolean useComparable;
+ public final boolean supportsInequality;
+
+ private TypeInfo(String className, String primitiveName, boolean useComparable, boolean supportsInequality) {
+ this.className = className;
+ this.primitiveName = primitiveName;
+ this.useComparable = useComparable;
+ this.supportsInequality = supportsInequality;
+ }
+ }
+
+ private static final TypeInfo[] TYPES = new TypeInfo[]{
+ new TypeInfo("Integer", "int", false, true),
+ new TypeInfo("Long", "long", false, true),
+ new TypeInfo("Boolean", "boolean", false, false),
+ new TypeInfo("Float", "float", false, true),
+ new TypeInfo("Double", "double", false, true),
+ new TypeInfo("Binary", "Binary", true, true),
+ };
+
+ public void run() throws IOException {
+ add("package org.apache.parquet.filter2.recordlevel;\n" +
+ "\n" +
+ "import org.apache.parquet.hadoop.metadata.ColumnPath;\n" +
+ "import org.apache.parquet.filter2.predicate.Operators.Eq;\n" +
+ "import org.apache.parquet.filter2.predicate.Operators.Gt;\n" +
+ "import org.apache.parquet.filter2.predicate.Operators.GtEq;\n" +
+ "import org.apache.parquet.filter2.predicate.Operators.LogicalNotUserDefined;\n" +
+ "import org.apache.parquet.filter2.predicate.Operators.Lt;\n" +
+ "import org.apache.parquet.filter2.predicate.Operators.LtEq;\n" +
+ "import org.apache.parquet.filter2.predicate.Operators.NotEq;\n" +
+ "import org.apache.parquet.filter2.predicate.Operators.UserDefined;\n" +
+ "import org.apache.parquet.filter2.predicate.UserDefinedPredicate;\n" +
+ "import org.apache.parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.ValueInspector;\n" +
+ "import org.apache.parquet.io.api.Binary;\n\n" +
+ "/**\n" +
+ " * This class is auto-generated by {@link parquet.filter2.IncrementallyUpdatedFilterPredicateGenerator}\n" +
+ " * Do not manually edit!\n" +
+ " * See {@link IncrementallyUpdatedFilterPredicateBuilderBase}\n" +
+ " */\n");
+
+ add("public class IncrementallyUpdatedFilterPredicateBuilder extends IncrementallyUpdatedFilterPredicateBuilderBase {\n\n");
+
+ addVisitBegin("Eq");
+ for (TypeInfo info : TYPES) {
+ addEqNotEqCase(info, true);
+ }
+ addVisitEnd();
+
+ addVisitBegin("NotEq");
+ for (TypeInfo info : TYPES) {
+ addEqNotEqCase(info, false);
+ }
+ addVisitEnd();
+
+ addVisitBegin("Lt");
+ for (TypeInfo info : TYPES) {
+ addInequalityCase(info, "<");
+ }
+ addVisitEnd();
+
+ addVisitBegin("LtEq");
+ for (TypeInfo info : TYPES) {
+ addInequalityCase(info, "<=");
+ }
+ addVisitEnd();
+
+ addVisitBegin("Gt");
+ for (TypeInfo info : TYPES) {
+ addInequalityCase(info, ">");
+ }
+ addVisitEnd();
+
+ addVisitBegin("GtEq");
+ for (TypeInfo info : TYPES) {
+ addInequalityCase(info, ">=");
+ }
+ addVisitEnd();
+
+ add(" @Override\n" +
+ " public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> IncrementallyUpdatedFilterPredicate visit(UserDefined<T, U> pred) {\n");
+ addUdpBegin();
+ for (TypeInfo info : TYPES) {
+ addUdpCase(info, false);
+ }
+ addVisitEnd();
+
+ add(" @Override\n" +
+ " public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> IncrementallyUpdatedFilterPredicate visit(LogicalNotUserDefined<T, U> notPred) {\n" +
+ " UserDefined<T, U> pred = notPred.getUserDefined();\n");
+ addUdpBegin();
+ for (TypeInfo info : TYPES) {
+ addUdpCase(info, true);
+ }
+ addVisitEnd();
+
+ add("}\n");
+ writer.close();
+ }
+
+ private void addVisitBegin(String inVar) throws IOException {
+ add(" @Override\n" +
+ " public <T extends Comparable<T>> IncrementallyUpdatedFilterPredicate visit(" + inVar + "<T> pred) {\n" +
+ " ColumnPath columnPath = pred.getColumn().getColumnPath();\n" +
+ " Class<T> clazz = pred.getColumn().getColumnType();\n" +
+ "\n" +
+ " ValueInspector valueInspector = null;\n\n");
+ }
+
+ private void addVisitEnd() throws IOException {
+ add(" if (valueInspector == null) {\n" +
+ " throw new IllegalArgumentException(\"Encountered unknown type \" + clazz);\n" +
+ " }\n" +
+ "\n" +
+ " addValueInspector(columnPath, valueInspector);\n" +
+ " return valueInspector;\n" +
+ " }\n\n");
+ }
+
+ private void addEqNotEqCase(TypeInfo info, boolean isEq) throws IOException {
+ add(" if (clazz.equals(" + info.className + ".class)) {\n" +
+ " if (pred.getValue() == null) {\n" +
+ " valueInspector = new ValueInspector() {\n" +
+ " @Override\n" +
+ " public void updateNull() {\n" +
+ " setResult(" + isEq + ");\n" +
+ " }\n" +
+ "\n" +
+ " @Override\n" +
+ " public void update(" + info.primitiveName + " value) {\n" +
+ " setResult(" + !isEq + ");\n" +
+ " }\n" +
+ " };\n" +
+ " } else {\n" +
+ " final " + info.primitiveName + " target = (" + info.className + ") (Object) pred.getValue();\n" +
+ "\n" +
+ " valueInspector = new ValueInspector() {\n" +
+ " @Override\n" +
+ " public void updateNull() {\n" +
+ " setResult(" + !isEq +");\n" +
+ " }\n" +
+ "\n" +
+ " @Override\n" +
+ " public void update(" + info.primitiveName + " value) {\n");
+
+ if (info.useComparable) {
+ add(" setResult(" + compareEquality("value", "target", isEq) + ");\n");
+ } else {
+ add(" setResult(" + (isEq ? "value == target" : "value != target" ) + ");\n");
+ }
+
+ add(" }\n" +
+ " };\n" +
+ " }\n" +
+ " }\n\n");
+ }
+
+ private void addInequalityCase(TypeInfo info, String op) throws IOException {
+ if (!info.supportsInequality) {
+ add(" if (clazz.equals(" + info.className + ".class)) {\n");
+ add(" throw new IllegalArgumentException(\"Operator " + op + " not supported for " + info.className + "\");\n");
+ add(" }\n\n");
+ return;
+ }
+
+ add(" if (clazz.equals(" + info.className + ".class)) {\n" +
+ " final " + info.primitiveName + " target = (" + info.className + ") (Object) pred.getValue();\n" +
+ "\n" +
+ " valueInspector = new ValueInspector() {\n" +
+ " @Override\n" +
+ " public void updateNull() {\n" +
+ " setResult(false);\n" +
+ " }\n" +
+ "\n" +
+ " @Override\n" +
+ " public void update(" + info.primitiveName + " value) {\n");
+
+ if (info.useComparable) {
+ add(" setResult(value.compareTo(target) " + op + " 0);\n");
+ } else {
+ add(" setResult(value " + op + " target);\n");
+ }
+ add(" }\n" +
+ " };\n" +
+ " }\n\n");
+ }
+
+ private void addUdpBegin() throws IOException {
+ add(" ColumnPath columnPath = pred.getColumn().getColumnPath();\n" +
+ " Class<T> clazz = pred.getColumn().getColumnType();\n" +
+ "\n" +
+ " ValueInspector valueInspector = null;\n" +
+ "\n" +
+ " final U udp = pred.getUserDefinedPredicate();\n" +
+ "\n");
+ }
+
+ private void addUdpCase(TypeInfo info, boolean invert)throws IOException {
+ add(" if (clazz.equals(" + info.className + ".class)) {\n" +
+ " valueInspector = new ValueInspector() {\n" +
+ " @Override\n" +
+ " public void updateNull() {\n" +
+ " setResult(" + (invert ? "!" : "") + "udp.keep(null));\n" +
+ " }\n" +
+ "\n" +
+ " @SuppressWarnings(\"unchecked\")\n" +
+ " @Override\n" +
+ " public void update(" + info.primitiveName + " value) {\n" +
+ " setResult(" + (invert ? "!" : "") + "udp.keep((T) (Object) value));\n" +
+ " }\n" +
+ " };\n" +
+ " }\n\n");
+ }
+
+ private String compareEquality(String var, String target, boolean eq) {
+ return var + ".compareTo(" + target + ")" + (eq ? " == 0 " : " != 0");
+ }
+
+ private void add(String s) throws IOException {
+ writer.write(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/parquet/encoding/Generator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/parquet/encoding/Generator.java b/parquet-generator/src/main/java/parquet/encoding/Generator.java
deleted file mode 100644
index 148d838..0000000
--- a/parquet-generator/src/main/java/parquet/encoding/Generator.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.encoding;
-
-import parquet.encoding.bitpacking.ByteBasedBitPackingGenerator;
-import parquet.encoding.bitpacking.IntBasedBitPackingGenerator;
-
-/**
- * main class for code generation hook in build for encodings generation
- *
- * @author Julien Le Dem
- *
- */
-public class Generator {
-
- public static void main(String[] args) throws Exception {
- IntBasedBitPackingGenerator.main(args);
- ByteBasedBitPackingGenerator.main(args);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java b/parquet-generator/src/main/java/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
deleted file mode 100644
index 5bcf4b0..0000000
--- a/parquet-generator/src/main/java/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.encoding.bitpacking;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-
-import parquet.bytes.BytesUtils;
-
-/**
- *
- * This class generates bit packers that pack the most significant bit first.
- * The result of the generation is checked in. To regenerate the code run this class and check in the result.
- *
- * TODO: remove the unnecessary masks for perf
- *
- * @author Julien Le Dem
- *
- */
-public class ByteBasedBitPackingGenerator {
-
- private static final String CLASS_NAME_PREFIX = "ByteBitPacking";
- private static final int PACKER_COUNT = 32;
-
- public static void main(String[] args) throws Exception {
- String basePath = args[0];
- generateScheme(CLASS_NAME_PREFIX + "BE", true, basePath);
- generateScheme(CLASS_NAME_PREFIX + "LE", false, basePath);
- }
-
- private static void generateScheme(String className, boolean msbFirst, String basePath) throws IOException {
- final File file = new File(basePath + "/parquet/column/values/bitpacking/" + className + ".java").getAbsoluteFile();
- if (!file.getParentFile().exists()) {
- file.getParentFile().mkdirs();
- }
- FileWriter fw = new FileWriter(file);
- fw.append("package parquet.column.values.bitpacking;\n");
- fw.append("\n");
- fw.append("/**\n");
- if (msbFirst) {
- fw.append(" * Packs from the Most Significant Bit first\n");
- } else {
- fw.append(" * Packs from the Least Significant Bit first\n");
- }
- fw.append(" * \n");
- fw.append(" * @author automatically generated\n");
- fw.append(" * @see ByteBasedBitPackingGenerator\n");
- fw.append(" *\n");
- fw.append(" */\n");
- fw.append("public abstract class " + className + " {\n");
- fw.append("\n");
- fw.append(" private static final BytePacker[] packers = new BytePacker[33];\n");
- fw.append(" static {\n");
- for (int i = 0; i <= PACKER_COUNT; i++) {
- fw.append(" packers[" + i + "] = new Packer" + i + "();\n");
- }
- fw.append(" }\n");
- fw.append("\n");
- fw.append(" public static final BytePackerFactory factory = new BytePackerFactory() {\n");
- fw.append(" public BytePacker newBytePacker(int bitWidth) {\n");
- fw.append(" return packers[bitWidth];\n");
- fw.append(" }\n");
- fw.append(" };\n");
- fw.append("\n");
- for (int i = 0; i <= PACKER_COUNT; i++) {
- generateClass(fw, i, msbFirst);
- fw.append("\n");
- }
- fw.append("}\n");
- fw.close();
- }
-
- private static void generateClass(FileWriter fw, int bitWidth, boolean msbFirst) throws IOException {
- fw.append(" private static final class Packer" + bitWidth + " extends BytePacker {\n");
- fw.append("\n");
- fw.append(" private Packer" + bitWidth + "() {\n");
- fw.append(" super("+bitWidth+");\n");
- fw.append(" }\n");
- fw.append("\n");
- // Packing
- generatePack(fw, bitWidth, 1, msbFirst);
- generatePack(fw, bitWidth, 4, msbFirst);
-
- // Unpacking
- generateUnpack(fw, bitWidth, 1, msbFirst);
- generateUnpack(fw, bitWidth, 4, msbFirst);
-
- fw.append(" }\n");
- }
-
- private static int getShift(FileWriter fw, int bitWidth, boolean msbFirst,
- int byteIndex, int valueIndex) throws IOException {
- // relative positions of the start and end of the value to the start and end of the byte
- int valueStartBitIndex = (valueIndex * bitWidth) - (8 * (byteIndex));
- int valueEndBitIndex = ((valueIndex + 1) * bitWidth) - (8 * (byteIndex + 1));
-
- // boundaries of the current value that we want
- int valueStartBitWanted;
- int valueEndBitWanted;
- // boundaries of the current byte that will receive them
- int byteStartBitWanted;
- int byteEndBitWanted;
-
- int shift;
-
- if (msbFirst) {
- valueStartBitWanted = valueStartBitIndex < 0 ? bitWidth - 1 + valueStartBitIndex : bitWidth - 1;
- valueEndBitWanted = valueEndBitIndex > 0 ? valueEndBitIndex : 0;
- byteStartBitWanted = valueStartBitIndex < 0 ? 8 : 7 - valueStartBitIndex;
- byteEndBitWanted = valueEndBitIndex > 0 ? 0 : -valueEndBitIndex;
- shift = valueEndBitWanted - byteEndBitWanted;
- } else {
- valueStartBitWanted = bitWidth - 1 - (valueEndBitIndex > 0 ? valueEndBitIndex : 0);
- valueEndBitWanted = bitWidth - 1 - (valueStartBitIndex < 0 ? bitWidth - 1 + valueStartBitIndex : bitWidth - 1);
- byteStartBitWanted = 7 - (valueEndBitIndex > 0 ? 0 : -valueEndBitIndex);
- byteEndBitWanted = 7 - (valueStartBitIndex < 0 ? 8 : 7 - valueStartBitIndex);
- shift = valueStartBitWanted - byteStartBitWanted;
- }
-
- visualizeAlignment(
- fw, bitWidth, valueEndBitIndex,
- valueStartBitWanted, valueEndBitWanted,
- byteStartBitWanted, byteEndBitWanted,
- shift
- );
- return shift;
- }
-
- private static void visualizeAlignment(FileWriter fw, int bitWidth,
- int valueEndBitIndex, int valueStartBitWanted, int valueEndBitWanted,
- int byteStartBitWanted, int byteEndBitWanted, int shift) throws IOException {
- // ASCII art to visualize what is happening
- fw.append("//");
- int buf = 2 + Math.max(0, bitWidth + 8);
- for (int i = 0; i < buf; i++) {
- fw.append(" ");
- }
- fw.append("[");
- for (int i = 7; i >= 0; i--) {
- if (i<=byteStartBitWanted && i>=byteEndBitWanted) {
- fw.append(String.valueOf(i));
- } else {
- fw.append("_");
- }
- }
- fw.append("]\n //");
- for (int i = 0; i < buf + (8 - bitWidth + shift); i++) {
- fw.append(" ");
- }
- fw.append("[");
- for (int i = bitWidth - 1; i >= 0 ; i--) {
- if (i<=valueStartBitWanted && i>=valueEndBitWanted) {
- fw.append(String.valueOf(i % 10));
- } else {
- fw.append("_");
- }
- }
- fw.append("]\n");
- fw.append(" ");
- }
-
- private static void generatePack(FileWriter fw, int bitWidth, int batch, boolean msbFirst) throws IOException {
- int mask = genMask(bitWidth);
- fw.append(" public final void pack" + (batch * 8) + "Values(final int[] in, final int inPos, final byte[] out, final int outPos) {\n");
- for (int byteIndex = 0; byteIndex < bitWidth * batch; ++byteIndex) {
- fw.append(" out[" + align(byteIndex, 2) + " + outPos] = (byte)((\n");
- int startIndex = (byteIndex * 8) / bitWidth;
- int endIndex = ((byteIndex + 1) * 8 + bitWidth - 1) / bitWidth;
- for (int valueIndex = startIndex; valueIndex < endIndex; valueIndex++) {
-
- if (valueIndex == startIndex) {
- fw.append(" ");
- } else {
- fw.append("\n | ");
- }
- int shift = getShift(fw, bitWidth, msbFirst, byteIndex, valueIndex);
-
- String shiftString = ""; // used when shift == 0
- if (shift > 0) {
- shiftString = " >>> " + shift;
- } else if (shift < 0) {
- shiftString = " << " + ( - shift);
- }
- fw.append("((in[" + align(valueIndex, 2) + " + inPos] & " + mask + ")" + shiftString + ")");
- }
- fw.append(") & 255);\n");
- }
- fw.append(" }\n");
- }
-
- private static void generateUnpack(FileWriter fw, int bitWidth, int batch, boolean msbFirst)
- throws IOException {
- fw.append(" public final void unpack" + (batch * 8) + "Values(final byte[] in, final int inPos, final int[] out, final int outPos) {\n");
- if (bitWidth > 0) {
- int mask = genMask(bitWidth);
- for (int valueIndex = 0; valueIndex < (batch * 8); ++valueIndex) {
- fw.append(" out[" + align(valueIndex, 2) + " + outPos] =\n");
-
- int startIndex = valueIndex * bitWidth / 8;
- int endIndex = BytesUtils.paddedByteCountFromBits((valueIndex + 1) * bitWidth);
-
- for (int byteIndex = startIndex; byteIndex < endIndex; byteIndex++) {
- if (byteIndex == startIndex) {
- fw.append(" ");
- } else {
- fw.append("\n | ");
- }
- int shift = getShift(fw, bitWidth, msbFirst, byteIndex, valueIndex);
-
- String shiftString = ""; // when shift == 0
- if (shift < 0) {
- shiftString = ">>> " + (-shift);
- } else if (shift > 0){
- shiftString = "<< " + shift;
- }
- fw.append(" (((((int)in[" + align(byteIndex, 2) + " + inPos]) & 255) " + shiftString + ") & " + mask + ")");
- }
- fw.append(";\n");
- }
- }
- fw.append(" }\n");
- }
-
- private static int genMask(int bitWidth) {
- int mask = 0;
- for (int i = 0; i < bitWidth; i++) {
- mask <<= 1;
- mask |= 1;
- }
- return mask;
- }
-
- private static String align(int value, int digits) {
- final String valueString = String.valueOf(value);
- StringBuilder result = new StringBuilder();
- for (int i = valueString.length(); i < digits; i++) {
- result.append(" ");
- }
- result.append(valueString);
- return result.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java b/parquet-generator/src/main/java/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java
deleted file mode 100644
index 153102a..0000000
--- a/parquet-generator/src/main/java/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.encoding.bitpacking;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-
-/**
- * Scheme designed by D. Lemire
- *
- * This is a re-implementation of The scheme released under Apache License Version 2.0
- * at https://github.com/lemire/JavaFastPFOR/blob/master/src/integercompression/BitPacking.java
- *
- * It generate two classes:
- * - LemireBitPackingLE, the original scheme, filling the LSB first
- * - LemireBitPackingBE, the scheme modified to fill the MSB first (and match our existing bit packing)
- *
- * The result of the generation is checked in. To regenerate the code run this class and check in the result.
- *
- * The generated classes pack the values into arrays of ints (as opposed to arrays of bytes) based on a given bit width.
- *
- * Note: This is not really used for now as the hadoop API does not really let write int[]. We need to revisit this
- *
- * @author Julien Le Dem
- *
- */
-public class IntBasedBitPackingGenerator {
-
- private static final String CLASS_NAME_PREFIX = "LemireBitPacking";
-
- public static void main(String[] args) throws Exception {
- String basePath = args[0];
- generateScheme(CLASS_NAME_PREFIX + "BE", true, basePath);
- generateScheme(CLASS_NAME_PREFIX + "LE", false, basePath);
- }
-
- private static void generateScheme(String className, boolean msbFirst, String basePath) throws IOException {
- final File file = new File(basePath + "/parquet/column/values/bitpacking/" + className + ".java").getAbsoluteFile();
- if (!file.getParentFile().exists()) {
- file.getParentFile().mkdirs();
- }
- FileWriter fw = new FileWriter(file);
- fw.append("package parquet.column.values.bitpacking;\n");
- fw.append("\n");
- fw.append("/**\n");
- fw.append(" * Based on the original implementation at at https://github.com/lemire/JavaFastPFOR/blob/master/src/integercompression/BitPacking.java\n");
- fw.append(" * Which is released under the\n");
- fw.append(" * Apache License Version 2.0 http://www.apache.org/licenses/.\n");
- fw.append(" * By Daniel Lemire, http://lemire.me/en/\n");
- fw.append(" * \n");
- fw.append(" * Scheme designed by D. Lemire\n");
- if (msbFirst) {
- fw.append(" * Adapted to pack from the Most Significant Bit first\n");
- }
- fw.append(" * \n");
- fw.append(" * @author automatically generated\n");
- fw.append(" * @see IntBasedBitPackingGenerator\n");
- fw.append(" *\n");
- fw.append(" */\n");
- fw.append("abstract class " + className + " {\n");
- fw.append("\n");
- fw.append(" private static final IntPacker[] packers = new IntPacker[32];\n");
- fw.append(" static {\n");
- for (int i = 0; i < 32; i++) {
- fw.append(" packers[" + i + "] = new Packer" + i + "();\n");
- }
- fw.append(" }\n");
- fw.append("\n");
- fw.append(" public static final IntPackerFactory factory = new IntPackerFactory() {\n");
- fw.append(" public IntPacker newIntPacker(int bitWidth) {\n");
- fw.append(" return packers[bitWidth];\n");
- fw.append(" }\n");
- fw.append(" };\n");
- fw.append("\n");
- for (int i = 0; i < 32; i++) {
- generateClass(fw, i, msbFirst);
- fw.append("\n");
- }
- fw.append("}\n");
- fw.close();
- }
-
- private static void generateClass(FileWriter fw, int bitWidth, boolean msbFirst) throws IOException {
- int mask = 0;
- for (int i = 0; i < bitWidth; i++) {
- mask <<= 1;
- mask |= 1;
- }
- fw.append(" private static final class Packer" + bitWidth + " extends IntPacker {\n");
- fw.append("\n");
- fw.append(" private Packer" + bitWidth + "() {\n");
- fw.append(" super(" + bitWidth + ");\n");
- fw.append(" }\n");
- fw.append("\n");
- // Packing
- fw.append(" public final void pack32Values(final int[] in, final int inPos, final int[] out, final int outPos) {\n");
- for (int i = 0; i < bitWidth; ++i) {
- fw.append(" out[" + align(i, 2) + " + outPos] =\n");
- int startIndex = (i * 32) / bitWidth;
- int endIndex = ((i + 1) * 32 + bitWidth - 1) / bitWidth;
- for (int j = startIndex; j < endIndex; j++) {
- if (j == startIndex) {
- fw.append(" ");
- } else {
- fw.append("\n | ");
- }
- String shiftString = getPackShiftString(bitWidth, i, startIndex, j, msbFirst);
- fw.append("((in[" + align(j, 2) + " + inPos] & " + mask + ")" + shiftString + ")");
- }
- fw.append(";\n");
- }
- fw.append(" }\n");
-
- // Unpacking
- fw.append(" public final void unpack32Values(final int[] in, final int inPos, final int[] out, final int outPos) {\n");
- if (bitWidth > 0) {
- for (int i = 0; i < 32; ++i) {
- fw.append(" out[" + align(i, 2) + " + outPos] =");
- int byteIndex = i * bitWidth / 32;
- String shiftString = getUnpackShiftString(bitWidth, i, msbFirst);
- fw.append(" ((in[" + align(byteIndex, 2) + " + inPos] " + shiftString + ") & " + mask + ")");
- if (((i + 1) * bitWidth - 1 ) / 32 != byteIndex) {
- // reading the end of the value from next int
- int bitsRead = ((i + 1) * bitWidth - 1) % 32 + 1;
- fw.append(" | ((in[" + align(byteIndex + 1, 2) + " + inPos]");
- if (msbFirst) {
- fw.append(") >>> " + align(32 - bitsRead, 2) + ")");
- } else {
- int lowerMask = 0;
- for (int j = 0; j < bitsRead; j++) {
- lowerMask <<= 1;
- lowerMask |= 1;
- }
- fw.append(" & " + lowerMask + ") << " + align(bitWidth - bitsRead, 2) + ")");
- }
- }
- fw.append(";\n");
- }
- }
- fw.append(" }\n");
- fw.append(" }\n");
- }
-
- private static String getUnpackShiftString(int bitWidth, int i, boolean msbFirst) {
- final int regularShift = i * bitWidth % 32;
- String shiftString;
- if (msbFirst) {
- int shift = 32 - (regularShift + bitWidth);
- if (shift < 0) {
- shiftString = "<< " + align(-shift, 2);
- } else {
- shiftString = ">>> " + align(shift, 2);
- }
- } else {
- shiftString = ">>> " + align(regularShift, 2);
- }
- return shiftString;
- }
-
- private static String getPackShiftString(int bitWidth, int integerIndex, int startIndex, int valueIndex, boolean msbFirst) {
- String shiftString;
- int regularShift = (valueIndex * bitWidth) % 32;
- if (msbFirst) { // filling most significant bit first
- int shift = 32 - (regularShift + bitWidth);
- if (valueIndex == startIndex && (integerIndex * 32) % bitWidth != 0) {
- // end of last value from previous int
- shiftString = " << " + align(32 - (((valueIndex + 1) * bitWidth) % 32), 2);
- } else if (shift < 0) {
- // partial last value
- shiftString = " >>> " + align(-shift, 2);
- } else {
- shiftString = " << " + align(shift, 2);
- }
- } else { // filling least significant bit first
- if (valueIndex == startIndex && (integerIndex * 32) % bitWidth != 0) {
- // end of last value from previous int
- shiftString = " >>> " + align(32 - regularShift, 2);
- } else {
- shiftString = " << " + align(regularShift, 2);
- }
- }
- return shiftString;
- }
-
- private static String align(int value, int digits) {
- final String valueString = String.valueOf(value);
- StringBuilder result = new StringBuilder();
- for (int i = valueString.length(); i < digits; i++) {
- result.append(" ");
- }
- result.append(valueString);
- return result.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/parquet/filter2/Generator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/parquet/filter2/Generator.java b/parquet-generator/src/main/java/parquet/filter2/Generator.java
deleted file mode 100644
index 9cc38bf..0000000
--- a/parquet-generator/src/main/java/parquet/filter2/Generator.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.filter2;
-
-/**
- * main class for code generation hook in build for filter2 generation
- */
-public class Generator {
- public static void main(String[] args) throws Exception {
- IncrementallyUpdatedFilterPredicateGenerator.main(args);
- }
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java b/parquet-generator/src/main/java/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java
deleted file mode 100644
index 9a705de..0000000
--- a/parquet-generator/src/main/java/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.filter2;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-
-public class IncrementallyUpdatedFilterPredicateGenerator {
-
- public static void main(String[] args) throws IOException {
- File srcFile = new File(args[0] + "/parquet/filter2/recordlevel/IncrementallyUpdatedFilterPredicateBuilder.java");
- srcFile = srcFile.getAbsoluteFile();
- File parent = srcFile.getParentFile();
- if (!parent.exists()) {
- if (!parent.mkdirs()) {
- throw new IOException("Couldn't mkdirs for " + parent);
- }
- }
- new IncrementallyUpdatedFilterPredicateGenerator(srcFile).run();
- }
-
- private final FileWriter writer;
-
- public IncrementallyUpdatedFilterPredicateGenerator(File file) throws IOException {
- this.writer = new FileWriter(file);
- }
-
- private static class TypeInfo {
- public final String className;
- public final String primitiveName;
- public final boolean useComparable;
- public final boolean supportsInequality;
-
- private TypeInfo(String className, String primitiveName, boolean useComparable, boolean supportsInequality) {
- this.className = className;
- this.primitiveName = primitiveName;
- this.useComparable = useComparable;
- this.supportsInequality = supportsInequality;
- }
- }
-
- private static final TypeInfo[] TYPES = new TypeInfo[]{
- new TypeInfo("Integer", "int", false, true),
- new TypeInfo("Long", "long", false, true),
- new TypeInfo("Boolean", "boolean", false, false),
- new TypeInfo("Float", "float", false, true),
- new TypeInfo("Double", "double", false, true),
- new TypeInfo("Binary", "Binary", true, true),
- };
-
- public void run() throws IOException {
- add("package parquet.filter2.recordlevel;\n" +
- "\n" +
- "import parquet.hadoop.metadata.ColumnPath;\n" +
- "import parquet.filter2.predicate.Operators.Eq;\n" +
- "import parquet.filter2.predicate.Operators.Gt;\n" +
- "import parquet.filter2.predicate.Operators.GtEq;\n" +
- "import parquet.filter2.predicate.Operators.LogicalNotUserDefined;\n" +
- "import parquet.filter2.predicate.Operators.Lt;\n" +
- "import parquet.filter2.predicate.Operators.LtEq;\n" +
- "import parquet.filter2.predicate.Operators.NotEq;\n" +
- "import parquet.filter2.predicate.Operators.UserDefined;\n" +
- "import parquet.filter2.predicate.UserDefinedPredicate;\n" +
- "import parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.ValueInspector;\n" +
- "import parquet.io.api.Binary;\n\n" +
- "/**\n" +
- " * This class is auto-generated by {@link parquet.filter2.IncrementallyUpdatedFilterPredicateGenerator}\n" +
- " * Do not manually edit!\n" +
- " * See {@link IncrementallyUpdatedFilterPredicateBuilderBase}\n" +
- " */\n");
-
- add("public class IncrementallyUpdatedFilterPredicateBuilder extends IncrementallyUpdatedFilterPredicateBuilderBase {\n\n");
-
- addVisitBegin("Eq");
- for (TypeInfo info : TYPES) {
- addEqNotEqCase(info, true);
- }
- addVisitEnd();
-
- addVisitBegin("NotEq");
- for (TypeInfo info : TYPES) {
- addEqNotEqCase(info, false);
- }
- addVisitEnd();
-
- addVisitBegin("Lt");
- for (TypeInfo info : TYPES) {
- addInequalityCase(info, "<");
- }
- addVisitEnd();
-
- addVisitBegin("LtEq");
- for (TypeInfo info : TYPES) {
- addInequalityCase(info, "<=");
- }
- addVisitEnd();
-
- addVisitBegin("Gt");
- for (TypeInfo info : TYPES) {
- addInequalityCase(info, ">");
- }
- addVisitEnd();
-
- addVisitBegin("GtEq");
- for (TypeInfo info : TYPES) {
- addInequalityCase(info, ">=");
- }
- addVisitEnd();
-
- add(" @Override\n" +
- " public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> IncrementallyUpdatedFilterPredicate visit(UserDefined<T, U> pred) {\n");
- addUdpBegin();
- for (TypeInfo info : TYPES) {
- addUdpCase(info, false);
- }
- addVisitEnd();
-
- add(" @Override\n" +
- " public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> IncrementallyUpdatedFilterPredicate visit(LogicalNotUserDefined<T, U> notPred) {\n" +
- " UserDefined<T, U> pred = notPred.getUserDefined();\n");
- addUdpBegin();
- for (TypeInfo info : TYPES) {
- addUdpCase(info, true);
- }
- addVisitEnd();
-
- add("}\n");
- writer.close();
- }
-
- private void addVisitBegin(String inVar) throws IOException {
- add(" @Override\n" +
- " public <T extends Comparable<T>> IncrementallyUpdatedFilterPredicate visit(" + inVar + "<T> pred) {\n" +
- " ColumnPath columnPath = pred.getColumn().getColumnPath();\n" +
- " Class<T> clazz = pred.getColumn().getColumnType();\n" +
- "\n" +
- " ValueInspector valueInspector = null;\n\n");
- }
-
- private void addVisitEnd() throws IOException {
- add(" if (valueInspector == null) {\n" +
- " throw new IllegalArgumentException(\"Encountered unknown type \" + clazz);\n" +
- " }\n" +
- "\n" +
- " addValueInspector(columnPath, valueInspector);\n" +
- " return valueInspector;\n" +
- " }\n\n");
- }
-
- private void addEqNotEqCase(TypeInfo info, boolean isEq) throws IOException {
- add(" if (clazz.equals(" + info.className + ".class)) {\n" +
- " if (pred.getValue() == null) {\n" +
- " valueInspector = new ValueInspector() {\n" +
- " @Override\n" +
- " public void updateNull() {\n" +
- " setResult(" + isEq + ");\n" +
- " }\n" +
- "\n" +
- " @Override\n" +
- " public void update(" + info.primitiveName + " value) {\n" +
- " setResult(" + !isEq + ");\n" +
- " }\n" +
- " };\n" +
- " } else {\n" +
- " final " + info.primitiveName + " target = (" + info.className + ") (Object) pred.getValue();\n" +
- "\n" +
- " valueInspector = new ValueInspector() {\n" +
- " @Override\n" +
- " public void updateNull() {\n" +
- " setResult(" + !isEq +");\n" +
- " }\n" +
- "\n" +
- " @Override\n" +
- " public void update(" + info.primitiveName + " value) {\n");
-
- if (info.useComparable) {
- add(" setResult(" + compareEquality("value", "target", isEq) + ");\n");
- } else {
- add(" setResult(" + (isEq ? "value == target" : "value != target" ) + ");\n");
- }
-
- add(" }\n" +
- " };\n" +
- " }\n" +
- " }\n\n");
- }
-
- private void addInequalityCase(TypeInfo info, String op) throws IOException {
- if (!info.supportsInequality) {
- add(" if (clazz.equals(" + info.className + ".class)) {\n");
- add(" throw new IllegalArgumentException(\"Operator " + op + " not supported for " + info.className + "\");\n");
- add(" }\n\n");
- return;
- }
-
- add(" if (clazz.equals(" + info.className + ".class)) {\n" +
- " final " + info.primitiveName + " target = (" + info.className + ") (Object) pred.getValue();\n" +
- "\n" +
- " valueInspector = new ValueInspector() {\n" +
- " @Override\n" +
- " public void updateNull() {\n" +
- " setResult(false);\n" +
- " }\n" +
- "\n" +
- " @Override\n" +
- " public void update(" + info.primitiveName + " value) {\n");
-
- if (info.useComparable) {
- add(" setResult(value.compareTo(target) " + op + " 0);\n");
- } else {
- add(" setResult(value " + op + " target);\n");
- }
- add(" }\n" +
- " };\n" +
- " }\n\n");
- }
-
- private void addUdpBegin() throws IOException {
- add(" ColumnPath columnPath = pred.getColumn().getColumnPath();\n" +
- " Class<T> clazz = pred.getColumn().getColumnType();\n" +
- "\n" +
- " ValueInspector valueInspector = null;\n" +
- "\n" +
- " final U udp = pred.getUserDefinedPredicate();\n" +
- "\n");
- }
-
- private void addUdpCase(TypeInfo info, boolean invert)throws IOException {
- add(" if (clazz.equals(" + info.className + ".class)) {\n" +
- " valueInspector = new ValueInspector() {\n" +
- " @Override\n" +
- " public void updateNull() {\n" +
- " setResult(" + (invert ? "!" : "") + "udp.keep(null));\n" +
- " }\n" +
- "\n" +
- " @SuppressWarnings(\"unchecked\")\n" +
- " @Override\n" +
- " public void update(" + info.primitiveName + " value) {\n" +
- " setResult(" + (invert ? "!" : "") + "udp.keep((T) (Object) value));\n" +
- " }\n" +
- " };\n" +
- " }\n\n");
- }
-
- private String compareEquality(String var, String target, boolean eq) {
- return var + ".compareTo(" + target + ")" + (eq ? " == 0 " : " != 0");
- }
-
- private void add(String s) throws IOException {
- writer.write(s);
- }
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-hadoop-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-hadoop-bundle/pom.xml b/parquet-hadoop-bundle/pom.xml
index 5ae94d4..a9f4620 100644
--- a/parquet-hadoop-bundle/pom.xml
+++ b/parquet-hadoop-bundle/pom.xml
@@ -18,7 +18,7 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
- <groupId>com.twitter</groupId>
+ <groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
<version>1.7.0-incubating-SNAPSHOT</version>
@@ -37,17 +37,17 @@
<dependencies>
<dependency>
- <groupId>com.twitter</groupId>
+ <groupId>org.apache.parquet</groupId>
<artifactId>parquet-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.twitter</groupId>
+ <groupId>org.apache.parquet</groupId>
<artifactId>parquet-column</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.twitter</groupId>
+ <groupId>org.apache.parquet</groupId>
<artifactId>parquet-hadoop</artifactId>
<version>${project.version}</version>
</dependency>
@@ -77,7 +77,7 @@
</filters>
<artifactSet>
<includes>
- <include>com.twitter:parquet-*</include>
+ <include>org.apache.parquet:parquet-*</include>
</includes>
</artifactSet>
</configuration>
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-hadoop-bundle/src/main/resources/org/apache/parquet/bundle
----------------------------------------------------------------------
diff --git a/parquet-hadoop-bundle/src/main/resources/org/apache/parquet/bundle b/parquet-hadoop-bundle/src/main/resources/org/apache/parquet/bundle
new file mode 100644
index 0000000..fe95886
--- /dev/null
+++ b/parquet-hadoop-bundle/src/main/resources/org/apache/parquet/bundle
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-hadoop-bundle/src/main/resources/parquet/bundle
----------------------------------------------------------------------
diff --git a/parquet-hadoop-bundle/src/main/resources/parquet/bundle b/parquet-hadoop-bundle/src/main/resources/parquet/bundle
deleted file mode 100644
index fe95886..0000000
--- a/parquet-hadoop-bundle/src/main/resources/parquet/bundle
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-hadoop/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml
index 970864f..f510401 100644
--- a/parquet-hadoop/pom.xml
+++ b/parquet-hadoop/pom.xml
@@ -18,7 +18,7 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
- <groupId>com.twitter</groupId>
+ <groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
<version>1.7.0-incubating-SNAPSHOT</version>
@@ -37,12 +37,12 @@
<dependencies>
<dependency>
- <groupId>com.twitter</groupId>
+ <groupId>org.apache.parquet</groupId>
<artifactId>parquet-column</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.twitter</groupId>
+ <groupId>org.apache.parquet</groupId>
<artifactId>parquet-format</artifactId>
<version>${parquet.format.version}</version>
</dependency>
@@ -59,7 +59,7 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>com.twitter</groupId>
+ <groupId>org.apache.parquet</groupId>
<artifactId>parquet-jackson</artifactId>
<version>${project.version}</version>
</dependency>