You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2015/05/06 19:31:16 UTC
spark git commit: [SPARK-6201] [SQL] promote string and do widen
types for IN
Repository: spark
Updated Branches:
refs/heads/master 150f671c2 -> c3eb441f5
[SPARK-6201] [SQL] promote string and do widen types for IN
huangjs
Acutally spark sql will first go through analysis period, in which we do widen types and promote strings, and then optimization, where constant IN will be converted into INSET.
So it turn out that we only need to fix this for IN.
Author: Daoyuan Wang <da...@intel.com>
Closes #4945 from adrian-wang/inset and squashes the following commits:
71e05cc [Daoyuan Wang] minor fix
581fa1c [Daoyuan Wang] mysql way
f3f7baf [Daoyuan Wang] address comments
5eed4bc [Daoyuan Wang] promote string and do widen types for IN
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c3eb441f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c3eb441f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c3eb441f
Branch: refs/heads/master
Commit: c3eb441f5487c9b6476e1d6e2a2d852dcc43b986
Parents: 150f671
Author: Daoyuan Wang <da...@intel.com>
Authored: Wed May 6 10:30:42 2015 -0700
Committer: Yin Huai <yh...@databricks.com>
Committed: Wed May 6 10:30:42 2015 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/HiveTypeCoercion.scala | 11 +++++++++++
.../apache/spark/sql/catalyst/optimizer/Optimizer.scala | 4 ++--
.../test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 9 +++++++++
3 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c3eb441f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
index 831fb4f..96e2aee 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
@@ -69,6 +69,7 @@ trait HiveTypeCoercion {
val typeCoercionRules =
PropagateTypes ::
ConvertNaNs ::
+ InConversion ::
WidenTypes ::
PromoteStrings ::
DecimalPrecision ::
@@ -287,6 +288,16 @@ trait HiveTypeCoercion {
}
}
+ /**
+ * Convert all expressions in in() list to the left operator type
+ */
+ object InConversion extends Rule[LogicalPlan] {
+ def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
+ case i @ In(a, b) if b.exists(_.dataType != a.dataType) =>
+ i.makeCopy(Array(a, b.map(Cast(_, a.dataType))))
+ }
+ }
+
// scalastyle:off
/**
* Calculates and propagates precision for fixed-precision decimals. Hive has a number of
http://git-wip-us.apache.org/repos/asf/spark/blob/c3eb441f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 709f7d6..e4a60f5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -310,8 +310,8 @@ object OptimizeIn extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
case q: LogicalPlan => q transformExpressionsDown {
case In(v, list) if !list.exists(!_.isInstanceOf[Literal]) =>
- val hSet = list.map(e => e.eval(null))
- InSet(v, HashSet() ++ hSet)
+ val hSet = list.map(e => e.eval(null))
+ InSet(v, HashSet() ++ hSet)
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/c3eb441f/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 0ab8558..208cec6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -120,6 +120,15 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
Row(1, 1) :: Nil)
}
+ test("SPARK-6201 IN type conversion") {
+ jsonRDD(sparkContext.parallelize(Seq("{\"a\": \"1\"}}", "{\"a\": \"2\"}}", "{\"a\": \"3\"}}")))
+ .registerTempTable("d")
+
+ checkAnswer(
+ sql("select * from d where d.a in (1,2)"),
+ Seq(Row("1"), Row("2")))
+ }
+
test("SPARK-3176 Added Parser of SQL ABS()") {
checkAnswer(
sql("SELECT ABS(-1.3)"),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org