You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by ju...@apache.org on 2017/05/12 21:40:34 UTC
parquet-mr git commit: PARQUET-969: Update parquet-tools to convert Decimal datatype to BigD…
Repository: parquet-mr
Updated Branches:
refs/heads/master 70f28810a -> a703ee75c
PARQUET-969: Update parquet-tools to convert Decimal datatype to BigD…
Update parquet-tools so that decimal datatypes in parquet files are converted to their actual number representation when cat'ing to stdout. Currently they are output in binary format.
Author: dsfcode <fo...@gmail.com>
Closes #412 from dsfcode/master and squashes the following commits:
7f05509 [dsfcode] PARQUET-969: Update parquet-tools to convert Decimal datatype to BigDecimal
Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/a703ee75
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/a703ee75
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/a703ee75
Branch: refs/heads/master
Commit: a703ee75c40e0207f6831c4d48e1c7e62f160305
Parents: 70f2881
Author: dsfcode <fo...@gmail.com>
Authored: Fri May 12 14:40:29 2017 -0700
Committer: Julien Le Dem <ju...@apache.org>
Committed: Fri May 12 14:40:29 2017 -0700
----------------------------------------------------------------------
.../tools/read/SimpleRecordConverter.java | 20 +++++++
.../tools/read/TestSimplePrimitiveRecord.java | 58 ++++++++++++++++++++
2 files changed, 78 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/a703ee75/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java
----------------------------------------------------------------------
diff --git a/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java b/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java
index 5f78444..29bb44f 100644
--- a/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java
+++ b/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java
@@ -22,6 +22,9 @@ import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.io.api.Converter;
import org.apache.parquet.io.api.GroupConverter;
@@ -70,6 +73,9 @@ public class SimpleRecordConverter extends GroupConverter {
case UTF8: return new StringConverter(field.getName());
case MAP_KEY_VALUE: break;
case ENUM: break;
+ case DECIMAL:
+ int scale = field.asPrimitiveType().getDecimalMetadata().getScale();
+ return new DecimalConverter(field.getName(), scale);
}
}
@@ -155,5 +161,19 @@ public class SimpleRecordConverter extends GroupConverter {
record.add(name, value.toStringUsingUTF8());
}
}
+
+ private class DecimalConverter extends SimplePrimitiveConverter {
+ private final int scale;
+
+ public DecimalConverter(String name, int scale) {
+ super(name);
+ this.scale = scale;
+ }
+
+ @Override
+ public void addBinary(Binary value) {
+ record.add(name, new BigDecimal(new BigInteger(value.getBytes()), scale));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/a703ee75/parquet-tools/src/test/java/org/apache/parquet/tools/read/TestSimplePrimitiveRecord.java
----------------------------------------------------------------------
diff --git a/parquet-tools/src/test/java/org/apache/parquet/tools/read/TestSimplePrimitiveRecord.java b/parquet-tools/src/test/java/org/apache/parquet/tools/read/TestSimplePrimitiveRecord.java
new file mode 100644
index 0000000..de27fcb
--- /dev/null
+++ b/parquet-tools/src/test/java/org/apache/parquet/tools/read/TestSimplePrimitiveRecord.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.tools.read;
+
+import org.junit.Assert;
+import org.junit.Test;
+import java.math.BigDecimal;
+
+public class TestSimplePrimitiveRecord {
+
+ class TestRecord {
+ private int x;
+ private int y;
+
+ public TestRecord(int x, int y) {
+ this.x = x;
+ this.y = y;
+ }
+
+ @Override
+ public String toString() {
+ return "TestRecord {" + x + "," + y + "}";
+ }
+ }
+
+ @Test
+ public void testBinary() {
+ SimpleMapRecord r = new SimpleMapRecord();
+ Assert.assertEquals("null", r.keyToString(null));
+ Assert.assertEquals("true", r.keyToString(true));
+ Assert.assertEquals("a", r.keyToString('a'));
+ Assert.assertEquals("3.0", r.keyToString(3.0));
+ Assert.assertEquals("4.0", r.keyToString(4.0f));
+ Assert.assertEquals("100", r.keyToString(100));
+ Assert.assertEquals("37", r.keyToString(37l));
+ Assert.assertEquals("-1", r.keyToString((short) -1));
+ Assert.assertEquals("test", r.keyToString("test"));
+ Assert.assertEquals("123.123", r.keyToString(new BigDecimal("123.123")));
+ }
+}
+