You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/02/23 05:49:29 UTC
spark git commit: [SPARK-23490][SQL] Check storage.locationUri with
existing table in CreateTable
Repository: spark
Updated Branches:
refs/heads/master c5abb3c2d -> 049f243c5
[SPARK-23490][SQL] Check storage.locationUri with existing table in CreateTable
## What changes were proposed in this pull request?
For CreateTable with Append mode, we should check if `storage.locationUri` is the same with existing table in `PreprocessTableCreation`
In the current code, there is only a simple exception if the `storage.locationUri` is different with existing table:
`org.apache.spark.sql.AnalysisException: Table or view not found:`
which can be improved.
## How was this patch tested?
Unit test
Author: Wang Gengliang <ge...@databricks.com>
Closes #20660 from gengliangwang/locationUri.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/049f243c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/049f243c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/049f243c
Branch: refs/heads/master
Commit: 049f243c59737699fee54fdc9d65cbd7c788032a
Parents: c5abb3c
Author: Wang Gengliang <ge...@databricks.com>
Authored: Thu Feb 22 21:49:25 2018 -0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Thu Feb 22 21:49:25 2018 -0800
----------------------------------------------------------------------
.../spark/sql/execution/datasources/rules.scala | 8 ++++++
.../spark/sql/execution/command/DDLSuite.scala | 29 ++++++++++++++++++++
2 files changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/049f243c/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
index 5cc21ee..0dea767 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
@@ -118,6 +118,14 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
s"`${existingProvider.getSimpleName}`. It doesn't match the specified format " +
s"`${specifiedProvider.getSimpleName}`.")
}
+ tableDesc.storage.locationUri match {
+ case Some(location) if location.getPath != existingTable.location.getPath =>
+ throw new AnalysisException(
+ s"The location of the existing table ${tableIdentWithDB.quotedString} is " +
+ s"`${existingTable.location}`. It doesn't match the specified location " +
+ s"`${tableDesc.location}`.")
+ case _ =>
+ }
if (query.schema.length != existingTable.schema.length) {
throw new AnalysisException(
http://git-wip-us.apache.org/repos/asf/spark/blob/049f243c/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index f76bfd2..b800e6f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -536,6 +536,35 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
}
}
+ test("create table - append to a non-partitioned table created with different paths") {
+ import testImplicits._
+ withTempDir { dir1 =>
+ withTempDir { dir2 =>
+ withTable("path_test") {
+ Seq(1L -> "a").toDF("v1", "v2")
+ .write
+ .mode(SaveMode.Append)
+ .format("json")
+ .option("path", dir1.getCanonicalPath)
+ .saveAsTable("path_test")
+
+ val ex = intercept[AnalysisException] {
+ Seq((3L, "c")).toDF("v1", "v2")
+ .write
+ .mode(SaveMode.Append)
+ .format("json")
+ .option("path", dir2.getCanonicalPath)
+ .saveAsTable("path_test")
+ }.getMessage
+ assert(ex.contains("The location of the existing table `default`.`path_test`"))
+
+ checkAnswer(
+ spark.table("path_test"), Row(1L, "a") :: Nil)
+ }
+ }
+ }
+ }
+
test("Refresh table after changing the data source table partitioning") {
import testImplicits._
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org