You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/11/21 00:34:07 UTC
spark git commit: [SPARK-4513][SQL] Support relational operator '<=>'
in Spark SQL
Repository: spark
Updated Branches:
refs/heads/master 1c53a5db9 -> 98e941978
[SPARK-4513][SQL] Support relational operator '<=>' in Spark SQL
The relational operator '<=>' is not working in Spark SQL. Same works in Spark HiveQL
Author: ravipesala <ra...@huawei.com>
Closes #3387 from ravipesala/<=> and squashes the following commits:
7198e90 [ravipesala] Supporting relational operator '<=>' in Spark SQL
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/98e94197
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/98e94197
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/98e94197
Branch: refs/heads/master
Commit: 98e9419784a9ad5096cfd563fa9a433786a90bd4
Parents: 1c53a5d
Author: ravipesala <ra...@huawei.com>
Authored: Thu Nov 20 15:34:03 2014 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Thu Nov 20 15:34:03 2014 -0800
----------------------------------------------------------------------
.../org/apache/spark/sql/catalyst/SparkSQLParser.scala | 2 +-
.../scala/org/apache/spark/sql/catalyst/SqlParser.scala | 1 +
.../test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 12 ++++++++++++
3 files changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/98e94197/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SparkSQLParser.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SparkSQLParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SparkSQLParser.scala
index f5c19ee..b198ed9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SparkSQLParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SparkSQLParser.scala
@@ -61,7 +61,7 @@ class SqlLexical(val keywords: Seq[String]) extends StdLexical {
delimiters += (
"@", "*", "+", "-", "<", "=", "<>", "!=", "<=", ">=", ">", "/", "(", ")",
- ",", ";", "%", "{", "}", ":", "[", "]", ".", "&", "|", "^", "~"
+ ",", ";", "%", "{", "}", ":", "[", "]", ".", "&", "|", "^", "~", "<=>"
)
override lazy val token: Parser[Token] =
http://git-wip-us.apache.org/repos/asf/spark/blob/98e94197/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
index affef27..dc1d349 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
@@ -234,6 +234,7 @@ class SqlParser extends AbstractSparkSQLParser {
| termExpression ~ (">=" ~> termExpression) ^^ { case e1 ~ e2 => GreaterThanOrEqual(e1, e2) }
| termExpression ~ ("!=" ~> termExpression) ^^ { case e1 ~ e2 => Not(EqualTo(e1, e2)) }
| termExpression ~ ("<>" ~> termExpression) ^^ { case e1 ~ e2 => Not(EqualTo(e1, e2)) }
+ | termExpression ~ ("<=>" ~> termExpression) ^^ { case e1 ~ e2 => EqualNullSafe(e1, e2) }
| termExpression ~ NOT.? ~ (BETWEEN ~> termExpression) ~ (AND ~> termExpression) ^^ {
case e ~ not ~ el ~ eu =>
val betweenExpr: Expression = And(GreaterThanOrEqual(e, el), LessThanOrEqual(e, eu))
http://git-wip-us.apache.org/repos/asf/spark/blob/98e94197/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index a635154..0a96831 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -973,4 +973,16 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("SELECT a.b + 1 FROM data GROUP BY a.b + 1"), 2)
dropTempTable("data")
}
+
+ test("Supporting relational operator '<=>' in Spark SQL") {
+ val nullCheckData1 = TestData(1,"1") :: TestData(2,null) :: Nil
+ val rdd1 = sparkContext.parallelize((0 to 1).map(i => nullCheckData1(i)))
+ rdd1.registerTempTable("nulldata1")
+ val nullCheckData2 = TestData(1,"1") :: TestData(2,null) :: Nil
+ val rdd2 = sparkContext.parallelize((0 to 1).map(i => nullCheckData2(i)))
+ rdd2.registerTempTable("nulldata2")
+ checkAnswer(sql("SELECT nulldata1.key FROM nulldata1 join " +
+ "nulldata2 on nulldata1.value <=> nulldata2.value"),
+ (1 to 2).map(i => Seq(i)))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org