You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2015/09/18 22:47:28 UTC
spark git commit: [SPARK-10449] [SQL] Don't merge decimal types with
incompatable precision or scales
Repository: spark
Updated Branches:
refs/heads/master c6f8135ee -> 3a22b1004
[SPARK-10449] [SQL] Don't merge decimal types with incompatable precision or scales
>From JIRA: Schema merging should only handle struct fields. But currently we also reconcile decimal precision and scale information.
Author: Holden Karau <ho...@pigscanfly.ca>
Closes #8634 from holdenk/SPARK-10449-dont-merge-different-precision.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3a22b100
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3a22b100
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3a22b100
Branch: refs/heads/master
Commit: 3a22b1004f527d54d399dd0225cd7f2f8ffad9c5
Parents: c6f8135
Author: Holden Karau <ho...@pigscanfly.ca>
Authored: Fri Sep 18 13:47:14 2015 -0700
Committer: Cheng Lian <li...@databricks.com>
Committed: Fri Sep 18 13:47:14 2015 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/types/StructType.scala | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/3a22b100/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
index b29cf22..d6b4367 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
@@ -373,10 +373,19 @@ object StructType extends AbstractDataType {
StructType(newFields)
case (DecimalType.Fixed(leftPrecision, leftScale),
- DecimalType.Fixed(rightPrecision, rightScale)) =>
- DecimalType(
- max(leftScale, rightScale) + max(leftPrecision - leftScale, rightPrecision - rightScale),
- max(leftScale, rightScale))
+ DecimalType.Fixed(rightPrecision, rightScale)) =>
+ if ((leftPrecision == rightPrecision) && (leftScale == rightScale)) {
+ DecimalType(leftPrecision, leftScale)
+ } else if ((leftPrecision != rightPrecision) && (leftScale != rightScale)) {
+ throw new SparkException("Failed to merge Decimal Tpes with incompatible " +
+ s"precision $leftPrecision and $rightPrecision & scale $leftScale and $rightScale")
+ } else if (leftPrecision != rightPrecision) {
+ throw new SparkException("Failed to merge Decimal Tpes with incompatible " +
+ s"precision $leftPrecision and $rightPrecision")
+ } else {
+ throw new SparkException("Failed to merge Decimal Tpes with incompatible " +
+ s"scala $leftScale and $rightScale")
+ }
case (leftUdt: UserDefinedType[_], rightUdt: UserDefinedType[_])
if leftUdt.userClass == rightUdt.userClass => leftUdt
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org