You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2022/01/21 12:21:40 UTC
[spark] branch master updated: [SPARK-37950][SQL] Take EXTERNAL as a reserved table property
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8ff48b3 [SPARK-37950][SQL] Take EXTERNAL as a reserved table property
8ff48b3 is described below
commit 8ff48b36cfa1fb10930c69a42887316b81e0b4a0
Author: PengLei <pe...@gmail.com>
AuthorDate: Fri Jan 21 20:20:09 2022 +0800
[SPARK-37950][SQL] Take EXTERNAL as a reserved table property
### What changes were proposed in this pull request?
Take `external` as a reserved table property. and do not allow use `external` for end-user when `spark.sql.legacy.notReserveProperties` == `false`.
### Why are the changes needed?
[#disscuss](https://github.com/apache/spark/pull/35204#issuecomment-1014752053).
keep it consistent with other properties like `location` `owner` `provider` and so on.
### Does this PR introduce _any_ user-facing change?
Yes. end-user could not use `external` as property key when create table with tblproperties and alter table set tblproperties.
### How was this patch tested?
existed testcase.
Closes #35268 from Peng-Lei/SPARK-37950.
Authored-by: PengLei <pe...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
docs/sql-migration-guide.md | 2 ++
.../main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala | 4 ++++
.../scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala | 3 ++-
.../spark/sql/execution/datasources/v2/ShowCreateTableExec.scala | 4 +---
.../spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala | 3 +--
.../test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 2 +-
6 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 5edf839..01c828a 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -56,6 +56,8 @@ license: |
- Since Spark 3.3, DESCRIBE FUNCTION fails if the function does not exist. In Spark 3.2 or earlier, DESCRIBE FUNCTION can still run and print "Function: func_name not found".
+ - Since Spark 3.3, the table property `external` becomes reserved. Certain commands will fail if you specify the `external` property, such as `CREATE TABLE ... TBLPROPERTIES` and `ALTER TABLE ... SET TBLPROPERTIES`. In Spark 3.2 and earlier, the table property `external` is silently ignored. You can set `spark.sql.legacy.notReserveProperties` to `true` to restore the old behavior.
+
## Upgrading from Spark SQL 3.1 to 3.2
- Since Spark 3.2, ADD FILE/JAR/ARCHIVE commands require each path to be enclosed by `"` or `'` if the path contains whitespaces.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 6a509db..35fe084 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -3203,6 +3203,10 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
throw QueryParsingErrors.cannotCleanReservedTablePropertyError(
PROP_OWNER, ctx, "it will be set to the current user")
case (PROP_OWNER, _) => false
+ case (PROP_EXTERNAL, _) if !legacyOn =>
+ throw QueryParsingErrors.cannotCleanReservedTablePropertyError(
+ PROP_EXTERNAL, ctx, "please use CREATE EXTERNAL TABLE")
+ case (PROP_EXTERNAL, _) => false
case _ => true
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
index 597b3c3..4092674 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
@@ -47,7 +47,8 @@ private[sql] object CatalogV2Util {
Seq(TableCatalog.PROP_COMMENT,
TableCatalog.PROP_LOCATION,
TableCatalog.PROP_PROVIDER,
- TableCatalog.PROP_OWNER)
+ TableCatalog.PROP_OWNER,
+ TableCatalog.PROP_EXTERNAL)
/**
* The list of reserved namespace properties, which can not be removed or changed directly by
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateTableExec.scala
index 8b3ad95..abe8d67 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateTableExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateTableExec.scala
@@ -127,9 +127,7 @@ case class ShowCreateTableExec(
val showProps = table.properties.asScala
.filterKeys(key => !CatalogV2Util.TABLE_RESERVED_PROPERTIES.contains(key)
&& !key.startsWith(TableCatalog.OPTION_PREFIX)
- && !tableOptions.contains(key)
- && !key.equals(TableCatalog.PROP_EXTERNAL)
- )
+ && !tableOptions.contains(key))
if (showProps.nonEmpty) {
val props = showProps.toSeq.sortBy(_._1).map {
case (key, value) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala
index 646eccb..1aa8e37 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala
@@ -785,8 +785,7 @@ class V2SessionCatalogTableSuite extends V2SessionCatalogBaseSuite {
private def filterV2TableProperties(
properties: util.Map[String, String]): Map[String, String] = {
properties.asScala.filter(kv => !CatalogV2Util.TABLE_RESERVED_PROPERTIES.contains(kv._1))
- .filter(!_._1.startsWith(TableCatalog.OPTION_PREFIX))
- .filter(_._1 != TableCatalog.PROP_EXTERNAL).toMap
+ .filter(!_._1.startsWith(TableCatalog.OPTION_PREFIX)).toMap
}
}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 85e3d0b5..e5bd1aa 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -844,7 +844,7 @@ class HiveDDLSuite
assert(
catalog.getTableMetadata(TableIdentifier(tabName)).tableType == CatalogTableType.MANAGED)
// The table property is case sensitive. Thus, external is allowed
- sql(s"ALTER TABLE $tabName SET TBLPROPERTIES ('external' = 'TRUE')")
+ sql(s"ALTER TABLE $tabName SET TBLPROPERTIES ('External' = 'TRUE')")
// The table type is not changed to external
assert(
catalog.getTableMetadata(TableIdentifier(tabName)).tableType == CatalogTableType.MANAGED)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org