You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2026/09/28 07:44:48 UTC
[L] Change in asterixdb[trinity]: [ASTERIXDB-3915][EXT]: Read wide Parquet decimals
From Hussain Towaileb <hu...@gmail.com>:
Hussain Towaileb has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21915?usp=email )
Change subject: [ASTERIXDB-3915][EXT]: Read wide Parquet decimals
......................................................................
[ASTERIXDB-3915][EXT]: Read wide Parquet decimals
With decimal-to-double enabled, a FIXED_LEN_BYTE_ARRAY or BINARY decimal
of precision 19 or 20, or one stored wider than its precision needs, was
decoded through a long and lost its high bytes: a DECIMAL(20,5) of
123.456 read as 0.0. The long path is now chosen by the stored width
rather than the precision.
Added unit and runtime test cases covering FIXED_LEN_BYTE_ARRAY decimals
of several precisions and widths.
Ext-ref: MB-74262
Generated-by: Claude Opus 5.5 <no...@anthropic.com>
Change-Id: I796e444a73721ce39b16824dc8fc3555d1aa0b5e
---
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/ParquetFileExampleGeneratorUtil.java
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/parquet-types/decimal-fixed-len-byte-array/decimal-fixed-len-byte-array.1.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/parquet-types/decimal-fixed-len-byte-array/decimal-fixed-len-byte-array.2.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/parquet-types/decimal-fixed-len-byte-array/decimal-fixed-len-byte-array.2.adm
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DecimalConverter.java
A asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/ParquetDecimalToDoubleTest.java
8 files changed, 264 insertions(+), 2 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/15/21915/1
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
index 7963132..cfe9492 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
@@ -272,6 +272,7 @@
loadData(generatedDataBasePath, "", "heterogeneous_1.parquet", definition, definitionSegment, false, false);
loadData(generatedDataBasePath, "", "heterogeneous_2.parquet", definition, definitionSegment, false, false);
loadData(generatedDataBasePath, "", "parquetTypes.parquet", definition, definitionSegment, false, false);
+ loadData(generatedDataBasePath, "", "parquetDecimals.parquet", definition, definitionSegment, false, false);
loadData(generatedDataBasePath, "", "friends.parquet", definition, definitionSegment, false, false);
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/ParquetFileExampleGeneratorUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/ParquetFileExampleGeneratorUtil.java
index 501fb27..71d5b01 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/ParquetFileExampleGeneratorUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/ParquetFileExampleGeneratorUtil.java
@@ -27,6 +27,7 @@
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneOffset;
+import java.util.Arrays;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
@@ -68,6 +69,7 @@
private static final int JULIAN_DAY_OF_EPOCH = 2440588;
private static final String FILE_NAME = "parquetTypes.parquet";
+ private static final String DECIMALS_FILE_NAME = "parquetDecimals.parquet";
private static final String SCHEMA = "message test { \n" + " required boolean boolean_field;\n"
+ " required int32 int8_field (INTEGER(8,true));\n"
@@ -99,6 +101,15 @@
+ " required group mapField (MAP) {\n" + " repeated group key_value {\n"
+ " required int32 key;\n" + " required int32 value;\n" + " }\n" + " }" + "}";
+ // An unscaled value of precision 19 or 20 needs 9 bytes, and d10_wide is wider than its precision needs
+ private static final String DECIMALS_SCHEMA = "message decimals {\n" + " required int32 id;\n"
+ + " required fixed_len_byte_array(4) d9 (DECIMAL(9,2));\n"
+ + " required fixed_len_byte_array(8) d18 (DECIMAL(18,5));\n"
+ + " required fixed_len_byte_array(9) d19 (DECIMAL(19,4));\n"
+ + " required fixed_len_byte_array(9) d20 (DECIMAL(20,5));\n"
+ + " required fixed_len_byte_array(16) d38 (DECIMAL(38,10));\n"
+ + " required fixed_len_byte_array(12) d10_wide (DECIMAL(10,2));\n" + "}";
+
private ParquetFileExampleGeneratorUtil() {
}
@@ -142,6 +153,41 @@
mapField.addGroup("key_value").append("key", 1).append("value", 1);
writer.write(message);
writer.close();
+ writeDecimalsExample(conf, root);
+ }
+
+ private static void writeDecimalsExample(Configuration conf, Path root) throws IOException {
+ MessageType schema = parseMessageType(DECIMALS_SCHEMA);
+ try (ParquetWriter<Group> writer =
+ ExampleParquetWriter.builder(new TestOutputFile(new Path(root, DECIMALS_FILE_NAME), conf))
+ .withType(schema).withCompressionCodec(UNCOMPRESSED).withConf(conf).build()) {
+ SimpleGroupFactory groupFactory = new SimpleGroupFactory(schema);
+ writer.write(decimalsRow(groupFactory, schema, 1, "1234567.89", "123.45600", "123.4560", "123.45600",
+ "123.4560000000", "99999999.99"));
+ writer.write(decimalsRow(groupFactory, schema, 2, "-1234567.89", "-9876543210.12345", "-123.4560",
+ "-123.45600", "-1.5000000000", "-99999999.99"));
+ writer.write(
+ decimalsRow(groupFactory, schema, 3, "9999999.99", "9999999999999.99999", "999999999999999.9999",
+ "-92233720368547.75809", "9999999999999999999999999999.9999999999", "0.01"));
+ }
+ }
+
+ private static Group decimalsRow(SimpleGroupFactory groupFactory, MessageType schema, int id, String... decimals) {
+ Group row = groupFactory.newGroup().append("id", id);
+ for (int i = 0; i < decimals.length; i++) {
+ int field = i + 1;
+ int width = schema.getType(field).asPrimitiveType().getTypeLength();
+ row.append(schema.getFieldName(field), toFixedLengthDecimal(new BigDecimal(decimals[i]), width));
+ }
+ return row;
+ }
+
+ private static Binary toFixedLengthDecimal(BigDecimal decimal, int width) {
+ byte[] minimal = decimal.unscaledValue().toByteArray();
+ byte[] bytes = new byte[width];
+ Arrays.fill(bytes, decimal.signum() < 0 ? (byte) 0xFF : 0);
+ System.arraycopy(minimal, 0, bytes, width - minimal.length, minimal.length);
+ return Binary.fromConstantByteArray(bytes);
}
private static int getSecondsSinceMidnight() {
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/parquet-types/decimal-fixed-len-byte-array/decimal-fixed-len-byte-array.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/parquet-types/decimal-fixed-len-byte-array/decimal-fixed-len-byte-array.1.ddl.sqlpp
new file mode 100644
index 0000000..59e401b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/parquet-types/decimal-fixed-len-byte-array/decimal-fixed-len-byte-array.1.ddl.sqlpp
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+
+CREATE TYPE ParquetType as {
+};
+
+CREATE EXTERNAL DATASET ParquetDataset(ParquetType) USING %adapter%
+(
+ %template%,
+ ("container"="playground"),
+ ("definition"="parquet-data/reviews"),
+ ("include"="*parquetDecimals.parquet"),
+ ("format" = "parquet"),
+ ("decimal-to-double" = "true")
+);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/parquet-types/decimal-fixed-len-byte-array/decimal-fixed-len-byte-array.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/parquet-types/decimal-fixed-len-byte-array/decimal-fixed-len-byte-array.2.query.sqlpp
new file mode 100644
index 0000000..4dda9b4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/parquet-types/decimal-fixed-len-byte-array/decimal-fixed-len-byte-array.2.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- param max-warnings:json=1000
+
+USE test;
+
+SELECT p.id, p.d9, p.d18, p.d19, p.d20, p.d38, p.d10_wide
+FROM ParquetDataset p
+ORDER BY p.id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/parquet-types/decimal-fixed-len-byte-array/decimal-fixed-len-byte-array.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/parquet-types/decimal-fixed-len-byte-array/decimal-fixed-len-byte-array.2.adm
new file mode 100644
index 0000000..08335cc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/parquet-types/decimal-fixed-len-byte-array/decimal-fixed-len-byte-array.2.adm
@@ -0,0 +1,3 @@
+{ "id": 1, "d9": 1234567.89, "d18": 123.456, "d19": 123.456, "d20": 123.456, "d38": 123.456, "d10_wide": 9.999999999E7 }
+{ "id": 2, "d9": -1234567.89, "d18": -9.87654321012345E9, "d19": -123.456, "d20": -123.456, "d38": -1.5, "d10_wide": -9.999999999E7 }
+{ "id": 3, "d9": 9999999.99, "d18": 1.0E13, "d19": 1.0E15, "d20": -9.223372036854777E13, "d38": 1.0E28, "d10_wide": 0.01 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 859a7e5..cf9735a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -205,6 +205,14 @@
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset" check-warnings="true">
+ <compilation-unit name="common/parquet/parquet-types/decimal-fixed-len-byte-array">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">common/parquet/parquet-types/decimal-fixed-len-byte-array</output-dir>
+ <source-location>false</source-location>
+ <expected-warn>Parquet decimal precision loss: precision '38' is greater than the maximum supported precision '20'</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset" check-warnings="true">
<compilation-unit name="common/parquet/parquet-types/temporal">
<placeholder name="adapter" value="S3" />
<output-dir compare="Text">common/parquet/parquet-types/temporal</output-dir>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DecimalConverter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DecimalConverter.java
index e93bcf7..c7d2377 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DecimalConverter.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/primitve/DecimalConverter.java
@@ -55,10 +55,11 @@
@Override
public void addBinary(Binary value) {
- if (precision <= LONG_MAX_PRECISION) {
+ // Decide by the stored width, not the precision: a precision of 19 or 20 needs 9 bytes, and a writer may
+ // use a FIXED_LEN_BYTE_ARRAY wider than the precision requires
+ if (value.length() <= Long.BYTES) {
addLong(getUnscaledLong(value.toByteBuffer()));
} else {
- //Unlimited precision
addConvertedDouble(new BigDecimal(new BigInteger(value.getBytes()), scale).doubleValue());
}
}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/ParquetDecimalToDoubleTest.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/ParquetDecimalToDoubleTest.java
new file mode 100644
index 0000000..eae8e1b
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/ParquetDecimalToDoubleTest.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.hdfs.parquet;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AObjectSerializerDeserializer;
+import org.apache.asterix.external.util.ExternalDataConstants.ParquetOptions;
+import org.apache.asterix.om.base.ADouble;
+import org.apache.asterix.om.base.ARecord;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.simple.SimpleGroupFactory;
+import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.util.HadoopOutputFile;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Types;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Reads FIXED_LEN_BYTE_ARRAY-backed decimals with {@link ParquetOptions#DECIMAL_TO_DOUBLE} enabled.
+ */
+public class ParquetDecimalToDoubleTest {
+ private static final Column D9 = new Column("d9", 9, 2, 4, "1234567.89", "-1234567.89", "9999999.99", "-0.01");
+ private static final Column D18 =
+ new Column("d18", 18, 5, 8, "123.45600", "-9876543210.12345", "9999999999999.99999", "-0.00001");
+ // an unscaled value of precision 19 or 20 needs 9 bytes, and may not fit in a long
+ private static final Column D20 =
+ new Column("d20", 20, 5, 9, "123.45600", "-123.45600", "999999999999999.99999", "-92233720368547.75809");
+ private static final Column D38 = new Column("d38", 38, 10, 16, "123.4560000000", "-1.5000000000",
+ "9999999999999999999999999999.9999999999", "-0.0000000001");
+ // wider than precision 10 needs, as some writers produce
+ private static final Column D10_WIDE =
+ new Column("d10_wide", 10, 2, 12, "99999999.99", "-99999999.99", "0.01", "-21474836.48");
+ private static final Column[] COLUMNS = { D9, D18, D20, D38, D10_WIDE };
+ private static final int ROW_COUNT = 4;
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ @Test
+ public void testFixedLenByteArrayDecimals() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(ParquetOptions.HADOOP_DECIMAL_TO_DOUBLE, "true");
+ conf.set(ParquetOptions.HADOOP_TIMEZONE, "");
+ Path path = new Path(new File(tempFolder.getRoot(), "decimals.parquet").toURI());
+ write(path, conf);
+
+ int row = 0;
+ try (ParquetReader<IValueReference> reader =
+ ParquetReader.builder(new ParquetReadSupport(), path).withConf(conf).build()) {
+ for (IValueReference value = reader.read(); value != null; value = reader.read()) {
+ Map<String, Double> actual = toDoubles(value);
+ for (Column column : COLUMNS) {
+ String decimal = column.values[row];
+ Assert.assertEquals(column.name + " = " + decimal, new BigDecimal(decimal).doubleValue(),
+ actual.get(column.name), 0.0);
+ }
+ row++;
+ }
+ }
+ Assert.assertEquals(ROW_COUNT, row);
+ }
+
+ private static void write(Path path, Configuration conf) throws Exception {
+ Types.MessageTypeBuilder builder = Types.buildMessage();
+ for (Column column : COLUMNS) {
+ builder.required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(column.width)
+ .as(LogicalTypeAnnotation.decimalType(column.scale, column.precision)).named(column.name);
+ }
+ MessageType schema = builder.named("decimals");
+ SimpleGroupFactory factory = new SimpleGroupFactory(schema);
+ try (ParquetWriter<Group> writer =
+ ExampleParquetWriter.builder(HadoopOutputFile.fromPath(path, conf)).withType(schema).build()) {
+ for (int row = 0; row < ROW_COUNT; row++) {
+ Group group = factory.newGroup();
+ for (Column column : COLUMNS) {
+ group.append(column.name, toFixedLength(new BigDecimal(column.values[row]), column.width));
+ }
+ writer.write(group);
+ }
+ }
+ }
+
+ private static Binary toFixedLength(BigDecimal decimal, int width) {
+ BigInteger unscaled = decimal.unscaledValue();
+ byte[] minimal = unscaled.toByteArray();
+ byte[] bytes = new byte[width];
+ Arrays.fill(bytes, unscaled.signum() < 0 ? (byte) 0xFF : 0);
+ System.arraycopy(minimal, 0, bytes, width - minimal.length, minimal.length);
+ return Binary.fromConstantByteArray(bytes);
+ }
+
+ private static Map<String, Double> toDoubles(IValueReference value) throws Exception {
+ DataInputStream in = new DataInputStream(
+ new ByteArrayInputStream(value.getByteArray(), value.getStartOffset(), value.getLength()));
+ ARecord record = (ARecord) AObjectSerializerDeserializer.INSTANCE.deserialize(in);
+ Map<String, Double> doubles = new HashMap<>();
+ for (int i = 0; i < record.numberOfFields(); i++) {
+ doubles.put(record.getType().getFieldNames()[i], ((ADouble) record.getValueByPos(i)).getDoubleValue());
+ }
+ return doubles;
+ }
+
+ private record Column(String name, int precision, int scale, int width, String... values) {
+ }
+}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21915?usp=email
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: trinity
Gerrit-Change-Id: I796e444a73721ce39b16824dc8fc3555d1aa0b5e
Gerrit-Change-Number: 21915
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <hu...@gmail.com>