You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2021/01/26 05:24:46 UTC
[spark] branch branch-3.1 updated: [SPARK-34229][SQL] Avro should
read decimal values with the file schema
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new f4340c4 [SPARK-34229][SQL] Avro should read decimal values with the file schema
f4340c4 is described below
commit f4340c4166c0f816de9eb13ef63cde5e60fffc81
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Tue Jan 26 14:23:54 2021 +0900
[SPARK-34229][SQL] Avro should read decimal values with the file schema
### What changes were proposed in this pull request?
This PR aims to fix Avro data source to use the decimal precision and scale of file schema.
### Why are the changes needed?
The decimal value should be interpreted with its original precision and scale. Otherwise, it returns incorrect result like the following. The schema mismatch happens when we use `userSpecifiedSchema` or there are multiple files with inconsistent schema or HiveMetastore schema is updated by the user.
```scala
scala> sql("SELECT 3.14 a").write.format("avro").save("/tmp/avro")
scala> spark.read.schema("a DECIMAL(4, 3)").format("avro").load("/tmp/avro").show
+-----+
| a|
+-----+
|0.314|
+-----+
```
### Does this PR introduce _any_ user-facing change?
Yes, this will return correct result.
### How was this patch tested?
Pass the CI with the newly added test case.
Closes #31329 from dongjoon-hyun/SPARK-34229.
Authored-by: Dongjoon Hyun <dh...@apple.com>
Signed-off-by: HyukjinKwon <gu...@apache.org>
(cherry picked from commit 7d09eac1ccb6a14a36fce30ae7cda575c29e1974)
Signed-off-by: HyukjinKwon <gu...@apache.org>
---
.../org/apache/spark/sql/avro/AvroDeserializer.scala | 16 ++++++++--------
.../test/scala/org/apache/spark/sql/avro/AvroSuite.scala | 8 ++++++++
2 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index c2c6f38..8ddc253 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -169,16 +169,16 @@ private[sql] class AvroDeserializer(
}
updater.set(ordinal, bytes)
- case (FIXED, d: DecimalType) => (updater, ordinal, value) =>
- val bigDecimal = decimalConversions.fromFixed(value.asInstanceOf[GenericFixed], avroType,
- LogicalTypes.decimal(d.precision, d.scale))
- val decimal = createDecimal(bigDecimal, d.precision, d.scale)
+ case (FIXED, _: DecimalType) => (updater, ordinal, value) =>
+ val d = avroType.getLogicalType.asInstanceOf[LogicalTypes.Decimal]
+ val bigDecimal = decimalConversions.fromFixed(value.asInstanceOf[GenericFixed], avroType, d)
+ val decimal = createDecimal(bigDecimal, d.getPrecision, d.getScale)
updater.setDecimal(ordinal, decimal)
- case (BYTES, d: DecimalType) => (updater, ordinal, value) =>
- val bigDecimal = decimalConversions.fromBytes(value.asInstanceOf[ByteBuffer], avroType,
- LogicalTypes.decimal(d.precision, d.scale))
- val decimal = createDecimal(bigDecimal, d.precision, d.scale)
+ case (BYTES, _: DecimalType) => (updater, ordinal, value) =>
+ val d = avroType.getLogicalType.asInstanceOf[LogicalTypes.Decimal]
+ val bigDecimal = decimalConversions.fromBytes(value.asInstanceOf[ByteBuffer], avroType, d)
+ val decimal = createDecimal(bigDecimal, d.getPrecision, d.getScale)
updater.setDecimal(ordinal, decimal)
case (RECORD, st: StructType) =>
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index 17bdeda..1ac0fb2 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -603,6 +603,14 @@ abstract class AvroSuite
}
}
+ test("SPARK-34229: Avro should read decimal values with the file schema") {
+ withTempPath { path =>
+ sql("SELECT 3.14 a").write.format("avro").save(path.toString)
+ val data = spark.read.schema("a DECIMAL(4, 3)").format("avro").load(path.toString).collect()
+ assert(data.map(_ (0)).contains(new java.math.BigDecimal("3.140")))
+ }
+ }
+
test("converting some specific sparkSQL types to avro") {
withTempPath { tempDir =>
val testSchema = StructType(Seq(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org