You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/07/03 09:25:06 UTC
spark git commit: [SPARK-8809][SQL] Remove ConvertNaNs analyzer rule.
Repository: spark
Updated Branches:
refs/heads/master 9b23e92c7 -> 2848f4da4
[SPARK-8809][SQL] Remove ConvertNaNs analyzer rule.
"NaN" from string to double is already handled by Cast expression itself.
Author: Reynold Xin <rx...@databricks.com>
Closes #7206 from rxin/convertnans and squashes the following commits:
3d99c33 [Reynold Xin] [SPARK-8809][SQL] Remove ConvertNaNs analyzer rule.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2848f4da
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2848f4da
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2848f4da
Branch: refs/heads/master
Commit: 2848f4da47d5c395de93ab9960bd905edfbd3439
Parents: 9b23e92
Author: Reynold Xin <rx...@databricks.com>
Authored: Fri Jul 3 00:25:02 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Fri Jul 3 00:25:02 2015 -0700
----------------------------------------------------------------------
.../catalyst/analysis/HiveTypeCoercion.scala | 33 --------------------
1 file changed, 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/2848f4da/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
index 6006e7b..38eb832 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
@@ -35,7 +35,6 @@ object HiveTypeCoercion {
val typeCoercionRules =
PropagateTypes ::
- ConvertNaNs ::
InConversion ::
WidenTypes ::
PromoteStrings ::
@@ -149,38 +148,6 @@ object HiveTypeCoercion {
}
/**
- * Converts string "NaN"s that are in binary operators with a NaN-able types (Float / Double) to
- * the appropriate numeric equivalent.
- */
- // TODO: remove this rule and make Cast handle Nan.
- object ConvertNaNs extends Rule[LogicalPlan] {
- private val StringNaN = Literal("NaN")
-
- def apply(plan: LogicalPlan): LogicalPlan = plan transform {
- case q: LogicalPlan => q transformExpressions {
- // Skip nodes who's children have not been resolved yet.
- case e if !e.childrenResolved => e
-
- /* Double Conversions */
- case b @ BinaryOperator(StringNaN, right @ DoubleType()) =>
- b.makeCopy(Array(Literal(Double.NaN), right))
- case b @ BinaryOperator(left @ DoubleType(), StringNaN) =>
- b.makeCopy(Array(left, Literal(Double.NaN)))
-
- /* Float Conversions */
- case b @ BinaryOperator(StringNaN, right @ FloatType()) =>
- b.makeCopy(Array(Literal(Float.NaN), right))
- case b @ BinaryOperator(left @ FloatType(), StringNaN) =>
- b.makeCopy(Array(left, Literal(Float.NaN)))
-
- /* Use float NaN by default to avoid unnecessary type widening */
- case b @ BinaryOperator(left @ StringNaN, StringNaN) =>
- b.makeCopy(Array(left, Literal(Float.NaN)))
- }
- }
- }
-
- /**
* Widens numeric types and converts strings to numbers when appropriate.
*
* Loosely based on rules from "Hadoop: The Definitive Guide" 2nd edition, by Tom White
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org