You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/03/08 05:53:37 UTC
spark git commit: [SPARK-23490][BACKPORT][SQL] Check
storage.locationUri with existing table in CreateTable
Repository: spark
Updated Branches:
refs/heads/branch-2.3 86ca91551 -> 1dd37ff3b
[SPARK-23490][BACKPORT][SQL] Check storage.locationUri with existing table in CreateTable
Backport #20660 to branch 2.3
=====================================
## What changes were proposed in this pull request?
For CreateTable with Append mode, we should check if `storage.locationUri` is the same with existing table in `PreprocessTableCreation`
In the current code, there is only a simple exception if the `storage.locationUri` is different with existing table:
`org.apache.spark.sql.AnalysisException: Table or view not found:`
which can be improved.
## How was this patch tested?
Unit test
Author: Wang Gengliang <ge...@databricks.com>
Closes #20766 from gengliangwang/backport_20660_to_2.3.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1dd37ff3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1dd37ff3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1dd37ff3
Branch: refs/heads/branch-2.3
Commit: 1dd37ff3b8c84c60858b159e745339ce19e53432
Parents: 86ca915
Author: Wang Gengliang <ge...@databricks.com>
Authored: Wed Mar 7 21:53:26 2018 -0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Wed Mar 7 21:53:26 2018 -0800
----------------------------------------------------------------------
.../spark/sql/execution/datasources/rules.scala | 8 ++++++
.../spark/sql/execution/command/DDLSuite.scala | 29 ++++++++++++++++++++
2 files changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/1dd37ff3/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
index 5cc21ee..0dea767 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
@@ -118,6 +118,14 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
s"`${existingProvider.getSimpleName}`. It doesn't match the specified format " +
s"`${specifiedProvider.getSimpleName}`.")
}
+ tableDesc.storage.locationUri match {
+ case Some(location) if location.getPath != existingTable.location.getPath =>
+ throw new AnalysisException(
+ s"The location of the existing table ${tableIdentWithDB.quotedString} is " +
+ s"`${existingTable.location}`. It doesn't match the specified location " +
+ s"`${tableDesc.location}`.")
+ case _ =>
+ }
if (query.schema.length != existingTable.schema.length) {
throw new AnalysisException(
http://git-wip-us.apache.org/repos/asf/spark/blob/1dd37ff3/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 63afcb5..1426df5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -536,6 +536,35 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
}
}
+ test("create table - append to a non-partitioned table created with different paths") {
+ import testImplicits._
+ withTempDir { dir1 =>
+ withTempDir { dir2 =>
+ withTable("path_test") {
+ Seq(1L -> "a").toDF("v1", "v2")
+ .write
+ .mode(SaveMode.Append)
+ .format("json")
+ .option("path", dir1.getCanonicalPath)
+ .saveAsTable("path_test")
+
+ val ex = intercept[AnalysisException] {
+ Seq((3L, "c")).toDF("v1", "v2")
+ .write
+ .mode(SaveMode.Append)
+ .format("json")
+ .option("path", dir2.getCanonicalPath)
+ .saveAsTable("path_test")
+ }.getMessage
+ assert(ex.contains("The location of the existing table `default`.`path_test`"))
+
+ checkAnswer(
+ spark.table("path_test"), Row(1L, "a") :: Nil)
+ }
+ }
+ }
+ }
+
test("Refresh table after changing the data source table partitioning") {
import testImplicits._
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org