[spark] branch master updated: [SPARK-40753][SQL] Fix bug in test case for catalog directory operation

[we...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 933dc0c42f0 [SPARK-40753][SQL] Fix bug in test case for catalog directory operation
933dc0c42f0 is described below

commit 933dc0c42f0caf74aaa077fd4f2c2e7208452b9b
Author: huangxiaoping <17...@qq.com>
AuthorDate: Wed Oct 19 16:57:05 2022 +0800

    [SPARK-40753][SQL] Fix bug in test case for catalog directory operation
    
    ### What changes were proposed in this pull request?
    Delete the directory that has been created in advance in the test case. Add the judgment of whether the directory exists.
    
    ### Why are the changes needed?
    The implementation class of ExternalCatalog will perform folder operations when performing operations such as create/drop database/table/partition. The test case creates a folder in advance when obtaining the DB/Partition path URI, in unit tests the external catalog does not actually create files, resulting in the result of the test case is not convincing enough force.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No
    
    ### How was this patch tested?
    
    tests were added
    
    Closes #38206 from huangxiaopingRD/fix_catalog_ut.
    
    Authored-by: huangxiaoping <17...@qq.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../sql/catalyst/catalog/ExternalCatalogSuite.scala  | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
index 8411d6320a7..32eb8849427 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
@@ -420,6 +420,9 @@ abstract class ExternalCatalogSuite extends SparkFunSuite {
     val partition2 =
       CatalogTablePartition(Map("partCol1" -> "3", "partCol2" -> "4"),
         storageFormat.copy(locationUri = Some(newLocationPart2)))
+    assert(!exists(newLocationPart1))
+    assert(!exists(newLocationPart2))
+
     catalog.createPartitions("db1", "tbl", Seq(partition1), ignoreIfExists = false)
     catalog.createPartitions("db1", "tbl", Seq(partition2), ignoreIfExists = false)
 
@@ -842,6 +845,7 @@ abstract class ExternalCatalogSuite extends SparkFunSuite {
   test("create/drop database should create/delete the directory") {
     val catalog = newBasicCatalog()
     val db = newDb("mydb")
+    assert(!exists(db.locationUri))
     catalog.createDatabase(db, ignoreIfExists = false)
     assert(exists(db.locationUri))
 
@@ -859,7 +863,7 @@ abstract class ExternalCatalogSuite extends SparkFunSuite {
       schema = new StructType().add("a", "int").add("b", "string"),
       provider = Some(defaultProvider)
     )
-
+    assert(!exists(db.locationUri, "my_table"))
     catalog.createTable(table, ignoreIfExists = false)
     assert(exists(db.locationUri, "my_table"))
 
@@ -1015,11 +1019,19 @@ abstract class CatalogTestUtils {
 
   def newFunc(): CatalogFunction = newFunc("funcName")
 
-  def newUriForDatabase(): URI = new URI(Utils.createTempDir().toURI.toString.stripSuffix("/"))
+  def newUriForDatabase(): URI = {
+    val file = Utils.createTempDir()
+    val uri = new URI(file.toURI.toString.stripSuffix("/"))
+    Utils.deleteRecursively(file)
+    uri
+  }
 
   def newUriForPartition(parts: Seq[String]): URI = {
-    val path = parts.foldLeft(Utils.createTempDir())(new java.io.File(_, _))
-    new URI(path.toURI.toString.stripSuffix("/"))
+    val file = Utils.createTempDir()
+    val path = parts.foldLeft(file)(new java.io.File(_, _))
+    val uri = new URI(path.toURI.toString.stripSuffix("/"))
+    Utils.deleteRecursively(file)
+    uri
   }
 
   def newDb(name: String): CatalogDatabase = {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org