You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2020/03/10 03:47:47 UTC
[spark] branch branch-3.0 updated: [SPARK-30902][SQL][FOLLOW-UP]
Allow ReplaceTableAsStatement to have none provider
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new b2f52ed [SPARK-30902][SQL][FOLLOW-UP] Allow ReplaceTableAsStatement to have none provider
b2f52ed is described below
commit b2f52ed8cbdfc4b7a1f161121cc9bf35973d140f
Author: Yuchen Huo <yu...@databricks.com>
AuthorDate: Tue Mar 10 11:37:31 2020 +0800
[SPARK-30902][SQL][FOLLOW-UP] Allow ReplaceTableAsStatement to have none provider
### What changes were proposed in this pull request?
This is a follow up for https://github.com/apache/spark/pull/27650 where allow None provider for create table. Here we are doing the same thing for ReplaceTable.
### Why are the changes needed?
Although currently the ASTBuilder doesn't seem to allow `replace` without `USING` clause. This would allow `DataFrameWriterV2` to use the statements instead of commands directly.
### Does this PR introduce any user-facing change?
No.
### How was this patch tested?
Existing tests
Closes #27838 from yuchenhuo/SPARK-30902.
Authored-by: Yuchen Huo <yu...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit a22994333ae129a3edaac75101fb4217fe8881c7)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala | 4 ++--
.../org/apache/spark/sql/catalyst/parser/AstBuilder.scala | 2 +-
.../apache/spark/sql/catalyst/plans/logical/statements.scala | 4 ++--
.../org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala | 4 ++--
.../spark/sql/catalyst/analysis/ResolveSessionCatalog.scala | 10 ++++++----
5 files changed, 13 insertions(+), 11 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
index 78a3171..88a3c0a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
@@ -153,7 +153,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
c.tableSchema,
// convert the bucket spec and add it as a transform
c.partitioning ++ c.bucketSpec.map(_.asTransform),
- convertTableProperties(c.properties, c.options, c.location, c.comment, Some(c.provider)),
+ convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
orCreate = c.orCreate)
case c @ ReplaceTableAsSelectStatement(
@@ -164,7 +164,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
// convert the bucket spec and add it as a transform
c.partitioning ++ c.bucketSpec.map(_.asTransform),
c.asSelect,
- convertTableProperties(c.properties, c.options, c.location, c.comment, Some(c.provider)),
+ convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
writeOptions = c.options,
orCreate = c.orCreate)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index ce3383d..3c23ecb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -2823,7 +2823,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
val (partitioning, bucketSpec, properties, options, location, comment) =
visitCreateTableClauses(ctx.createTableClauses())
val schema = Option(ctx.colTypeList()).map(createSchema)
- val provider = ctx.tableProvider.multipartIdentifier.getText
+ val provider = Option(ctx.tableProvider).map(_.multipartIdentifier.getText)
val orCreate = ctx.replaceTableHeader().CREATE() != null
Option(ctx.query).map(plan) match {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
index 89e1539..50c3cd0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
@@ -114,7 +114,7 @@ case class ReplaceTableStatement(
partitioning: Seq[Transform],
bucketSpec: Option[BucketSpec],
properties: Map[String, String],
- provider: String,
+ provider: Option[String],
options: Map[String, String],
location: Option[String],
comment: Option[String],
@@ -129,7 +129,7 @@ case class ReplaceTableAsSelectStatement(
partitioning: Seq[Transform],
bucketSpec: Option[BucketSpec],
properties: Map[String, String],
- provider: String,
+ provider: Option[String],
options: Map[String, String],
location: Option[String],
comment: Option[String],
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index 967d0bd..e3570899 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -2065,7 +2065,7 @@ class DDLParserSuite extends AnalysisTest {
replace.partitioning,
replace.bucketSpec,
replace.properties,
- Some(replace.provider),
+ replace.provider,
replace.options,
replace.location,
replace.comment)
@@ -2087,7 +2087,7 @@ class DDLParserSuite extends AnalysisTest {
rtas.partitioning,
rtas.bucketSpec,
rtas.properties,
- Some(rtas.provider),
+ rtas.provider,
rtas.options,
rtas.location,
rtas.comment)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index 90a3fb8..5330612 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -282,7 +282,8 @@ class ResolveSessionCatalog(
// session catalog and the table provider is not v2.
case c @ ReplaceTableStatement(
SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
- if (!isV2Provider(c.provider)) {
+ val provider = c.provider.getOrElse(conf.defaultDataSourceName)
+ if (!isV2Provider(provider)) {
throw new AnalysisException("REPLACE TABLE is only supported with v2 tables.")
} else {
ReplaceTable(
@@ -291,13 +292,14 @@ class ResolveSessionCatalog(
c.tableSchema,
// convert the bucket spec and add it as a transform
c.partitioning ++ c.bucketSpec.map(_.asTransform),
- convertTableProperties(c.properties, c.options, c.location, c.comment, Some(c.provider)),
+ convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
orCreate = c.orCreate)
}
case c @ ReplaceTableAsSelectStatement(
SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
- if (!isV2Provider(c.provider)) {
+ val provider = c.provider.getOrElse(conf.defaultDataSourceName)
+ if (!isV2Provider(provider)) {
throw new AnalysisException("REPLACE TABLE AS SELECT is only supported with v2 tables.")
} else {
ReplaceTableAsSelect(
@@ -306,7 +308,7 @@ class ResolveSessionCatalog(
// convert the bucket spec and add it as a transform
c.partitioning ++ c.bucketSpec.map(_.asTransform),
c.asSelect,
- convertTableProperties(c.properties, c.options, c.location, c.comment, Some(c.provider)),
+ convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
writeOptions = c.options,
orCreate = c.orCreate)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org