You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by hv...@apache.org on 2016/11/07 20:33:06 UTC
spark git commit: [SPARK-17108][SQL] Fix BIGINT and INT comparison
failure in spark sql
Repository: spark
Updated Branches:
refs/heads/master b06c23db9 -> 0d95662e7
[SPARK-17108][SQL] Fix BIGINT and INT comparison failure in spark sql
## What changes were proposed in this pull request?
Add a function to check if two integers are compatible when invoking `acceptsType()` in `DataType`.
## How was this patch tested?
Manually.
E.g.
```
spark.sql("create table t3(a map<bigint, array<string>>)")
spark.sql("select * from t3 where a[1] is not null")
```
Before:
```
cannot resolve 't.`a`[1]' due to data type mismatch: argument 2 requires bigint type, however, '1' is of int type.; line 1 pos 22
org.apache.spark.sql.AnalysisException: cannot resolve 't.`a`[1]' due to data type mismatch: argument 2 requires bigint type, however, '1' is of int type.; line 1 pos 22
at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:82)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:74)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:307)
```
After:
Run the sql queries above. No errors.
Author: Weiqing Yang <ya...@gmail.com>
Closes #15448 from weiqingy/SPARK_17108.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0d95662e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0d95662e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0d95662e
Branch: refs/heads/master
Commit: 0d95662e7fff26669d4f70e88fdac7a4128a4f49
Parents: b06c23d
Author: Weiqing Yang <ya...@gmail.com>
Authored: Mon Nov 7 21:33:01 2016 +0100
Committer: Herman van Hovell <hv...@databricks.com>
Committed: Mon Nov 7 21:33:01 2016 +0100
----------------------------------------------------------------------
.../catalyst/expressions/complexTypeExtractors.scala | 2 +-
.../apache/spark/sql/hive/execution/SQLQuerySuite.scala | 12 ++++++++++++
2 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0d95662e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
index abb5594..0c256c3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
@@ -260,7 +260,7 @@ case class GetArrayItem(child: Expression, ordinal: Expression)
* We need to do type checking here as `key` expression maybe unresolved.
*/
case class GetMapValue(child: Expression, key: Expression)
- extends BinaryExpression with ExpectsInputTypes with ExtractValue {
+ extends BinaryExpression with ImplicitCastInputTypes with ExtractValue {
private def keyType = child.dataType.asInstanceOf[MapType].keyType
http://git-wip-us.apache.org/repos/asf/spark/blob/0d95662e/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 5e08ef3..c21db35 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -1939,6 +1939,18 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}
}
+
+ test("SPARK-17108: Fix BIGINT and INT comparison failure in spark sql") {
+ sql("create table t1(a map<bigint, array<string>>)")
+ sql("select * from t1 where a[1] is not null")
+
+ sql("create table t2(a map<int, array<string>>)")
+ sql("select * from t2 where a[1] is not null")
+
+ sql("create table t3(a map<bigint, array<string>>)")
+ sql("select * from t3 where a[1L] is not null")
+ }
+
test("SPARK-17796 Support wildcard character in filename for LOAD DATA LOCAL INPATH") {
withTempDir { dir =>
for (i <- 1 to 3) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org