You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2019/03/11 06:02:43 UTC
[spark] branch master updated: [SPARK-27096][SQL] Reconcile the
join types between data frame and sql interface
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1b9fd67 [SPARK-27096][SQL] Reconcile the join types between data frame and sql interface
1b9fd67 is described below
commit 1b9fd67904671ea08526bfb7a97d694815d47665
Author: Dilip Biswal <db...@us.ibm.com>
AuthorDate: Mon Mar 11 14:02:21 2019 +0800
[SPARK-27096][SQL] Reconcile the join types between data frame and sql interface
## What changes were proposed in this pull request?
Currently in the grammar file, we have the joinType rule defined as following :
```
joinType
: INNER?
....
....
| LEFT SEMI
| LEFT? ANTI
;
```
The keyword LEFT is optional for ANTI join even though its not optional for SEMI join. When
using data frame interface join type "anti" is not allowed. The allowed types are "left_anti" or
"leftanti" for anti joins. ~~In this PR, i am making the LEFT keyword mandatory for ANTI joins so
it aligns better with the LEFT SEMI join in the grammar file and also the join types allowed from dataframe api.~~
This PR makes LEFT optional for SEMI join in .g4 and add "semi" and "anti" join types from dataframe.
~~I have not opened any JIRA for this as probably we may need some discussion to see if
we are going to address this or not.~~
## How was this patch tested?
Modified the join type tests.
Closes #23982 from dilipbiswal/join_fix.
Authored-by: Dilip Biswal <db...@us.ibm.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 2 +-
.../org/apache/spark/sql/catalyst/plans/joinTypes.scala | 8 ++++----
.../apache/spark/sql/catalyst/parser/PlanParserSuite.scala | 1 +
.../org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala | 2 ++
sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala | 4 ++--
.../test/resources/sql-tests/results/describe-query.sql.out | 2 +-
.../test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala | 12 ++++++++++++
.../src/test/scala/org/apache/spark/sql/DatasetSuite.scala | 12 +++++++++++-
8 files changed, 34 insertions(+), 9 deletions(-)
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index d2164ec..c172799 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -477,7 +477,7 @@ joinType
: INNER?
| CROSS
| LEFT OUTER?
- | LEFT SEMI
+ | LEFT? SEMI
| RIGHT OUTER?
| FULL OUTER?
| LEFT? ANTI
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala
index 86cdc26..feea1d2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala
@@ -27,8 +27,8 @@ object JoinType {
case "outer" | "full" | "fullouter" => FullOuter
case "leftouter" | "left" => LeftOuter
case "rightouter" | "right" => RightOuter
- case "leftsemi" => LeftSemi
- case "leftanti" => LeftAnti
+ case "leftsemi" | "semi" => LeftSemi
+ case "leftanti" | "anti" => LeftAnti
case "cross" => Cross
case _ =>
val supported = Seq(
@@ -36,8 +36,8 @@ object JoinType {
"outer", "full", "fullouter", "full_outer",
"leftouter", "left", "left_outer",
"rightouter", "right", "right_outer",
- "leftsemi", "left_semi",
- "leftanti", "left_anti",
+ "leftsemi", "left_semi", "semi",
+ "leftanti", "left_anti", "anti",
"cross")
throw new IllegalArgumentException(s"Unsupported join type '$typ'. " +
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
index f5da90f..d628150 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
@@ -370,6 +370,7 @@ class PlanParserSuite extends AnalysisTest {
test("full join", FullOuter, testAll)
test("full outer join", FullOuter, testAll)
test("left semi join", LeftSemi, testExistence)
+ test("semi join", LeftSemi, testExistence)
test("left anti join", LeftAnti, testExistence)
test("anti join", LeftAnti, testExistence)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala
index d56f979..43221bf 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala
@@ -48,11 +48,13 @@ class JoinTypesTest extends SparkFunSuite {
test("construct a LeftSemi type") {
assert(JoinType("leftsemi") === LeftSemi)
assert(JoinType("left_semi") === LeftSemi)
+ assert(JoinType("semi") === LeftSemi)
}
test("construct a LeftAnti type") {
assert(JoinType("leftanti") === LeftAnti)
assert(JoinType("left_anti") === LeftAnti)
+ assert(JoinType("anti") === LeftAnti)
}
test("construct a Cross type") {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index bd1ae50..24f5c81 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -929,7 +929,7 @@ class Dataset[T] private[sql](
* @param usingColumns Names of the columns to join on. This columns must exist on both sides.
* @param joinType Type of join to perform. Default `inner`. Must be one of:
* `inner`, `cross`, `outer`, `full`, `full_outer`, `left`, `left_outer`,
- * `right`, `right_outer`, `left_semi`, `left_anti`.
+ * `right`, `right_outer`, `left_semi`, `semi`, `left_anti`, `anti`.
*
* @note If you perform a self-join using this function without aliasing the input
* `DataFrame`s, you will NOT be able to reference any columns after the join, since
@@ -987,7 +987,7 @@ class Dataset[T] private[sql](
* @param joinExprs Join expression.
* @param joinType Type of join to perform. Default `inner`. Must be one of:
* `inner`, `cross`, `outer`, `full`, `full_outer`, `left`, `left_outer`,
- * `right`, `right_outer`, `left_semi`, `left_anti`.
+ * `right`, `right_outer`, `left_semi`, `semi`, `left_anti`, `anti`.
*
* @group untypedrel
* @since 2.0.0
diff --git a/sql/core/src/test/resources/sql-tests/results/describe-query.sql.out b/sql/core/src/test/resources/sql-tests/results/describe-query.sql.out
index 36cb314..9209ce1 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe-query.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe-query.sql.out
@@ -145,7 +145,7 @@ struct<>
-- !query 13 output
org.apache.spark.sql.catalyst.parser.ParseException
-mismatched input 'insert' expecting {<EOF>, '(', ',', 'SELECT', 'WHERE', 'GROUP', 'ORDER', 'HAVING', 'LIMIT', 'JOIN', 'CROSS', 'INNER', 'LEFT', 'RIGHT', 'FULL', 'NATURAL', 'PIVOT', 'LATERAL', 'WINDOW', 'UNION', 'EXCEPT', 'MINUS', 'INTERSECT', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'ANTI'}(line 3, pos 5)
+mismatched input 'insert' expecting {<EOF>, '(', ',', 'SELECT', 'WHERE', 'GROUP', 'ORDER', 'HAVING', 'LIMIT', 'JOIN', 'CROSS', 'INNER', 'LEFT', 'SEMI', 'RIGHT', 'FULL', 'NATURAL', 'PIVOT', 'LATERAL', 'WINDOW', 'UNION', 'EXCEPT', 'MINUS', 'INTERSECT', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'ANTI'}(line 3, pos 5)
== SQL ==
DESCRIBE
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
index 6bd12cb..ba120dc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
@@ -84,6 +84,18 @@ class DataFrameJoinSuite extends QueryTest with SharedSQLContext {
checkAnswer(
df.join(df2, Seq("int", "str"), "left_semi"),
Row(1, "1", 2) :: Nil)
+
+ checkAnswer(
+ df.join(df2, Seq("int", "str"), "semi"),
+ Row(1, "1", 2) :: Nil)
+
+ checkAnswer(
+ df.join(df2, Seq("int", "str"), "left_anti"),
+ Row(3, "3", 4) :: Nil)
+
+ checkAnswer(
+ df.join(df2, Seq("int", "str"), "anti"),
+ Row(3, "3", 4) :: Nil)
}
test("join - join using self join") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index 4eef115..db9f251 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -470,9 +470,19 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
assert(e1.contains("Invalid join type in joinWith: " + LeftSemi.sql))
val e2 = intercept[AnalysisException] {
+ ds1.joinWith(ds2, $"a.value" === $"b.value", "semi")
+ }.getMessage
+ assert(e2.contains("Invalid join type in joinWith: " + LeftSemi.sql))
+
+ val e3 = intercept[AnalysisException] {
ds1.joinWith(ds2, $"a.value" === $"b.value", "left_anti")
}.getMessage
- assert(e2.contains("Invalid join type in joinWith: " + LeftAnti.sql))
+ assert(e3.contains("Invalid join type in joinWith: " + LeftAnti.sql))
+
+ val e4 = intercept[AnalysisException] {
+ ds1.joinWith(ds2, $"a.value" === $"b.value", "anti")
+ }.getMessage
+ assert(e4.contains("Invalid join type in joinWith: " + LeftAnti.sql))
}
test("groupBy function, keys") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org