[spark] branch branch-2.4 updated: [SPARK-31163][SQL] TruncateTableCommand with acl/permission should handle non-existed path

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
     new 26ad3fe  [SPARK-31163][SQL] TruncateTableCommand with acl/permission should handle non-existed path
26ad3fe is described below

commit 26ad3fe2604506a2cbc5de0ecd1b5771f3965cb4
Author: yi.wu <yi...@databricks.com>
AuthorDate: Mon Mar 16 11:45:25 2020 -0700

    [SPARK-31163][SQL] TruncateTableCommand with acl/permission should handle non-existed path
    
    ### What changes were proposed in this pull request?
    
    This fix #26956
    Wrap try-catch on `fs.getFileStatus(path)` within acl/permission in case of the path doesn't exist.
    
    ### Why are the changes needed?
    
    `truncate table` may fail to re-create path in case of interruption or something else. As a result, next time we `truncate table` on the same table with acl/permission, it will fail due to `FileNotFoundException`. And it also brings behavior change compares to previous Spark version, which could still `truncate table` successfully even if the path doesn't exist.
    
    ### Does this PR introduce any user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Added UT.
    
    Closes #27923 from Ngone51/fix_truncate.
    
    Authored-by: yi.wu <yi...@databricks.com>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
    (cherry picked from commit cb26f636b08aea4c5c6bf5035a359cd3cbf335c0)
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../apache/spark/sql/execution/command/tables.scala |  2 +-
 .../spark/sql/execution/command/DDLSuite.scala      | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index 28dc4a4..8f504d3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -470,8 +470,8 @@ case class TruncateTableCommand(
           var optPermission: Option[FsPermission] = None
           var optAcls: Option[java.util.List[AclEntry]] = None
           if (!ignorePermissionAcl) {
-            val fileStatus = fs.getFileStatus(path)
             try {
+              val fileStatus = fs.getFileStatus(path)
               optPermission = Some(fileStatus.getPermission())
             } catch {
               case NonFatal(_) => // do nothing
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 0aabe86..73565f2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -2009,6 +2009,27 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     }
   }
 
+  test("SPARK-31163: acl/permission should handle non-existed path when truncating table") {
+    withSQLConf(SQLConf.TRUNCATE_TABLE_IGNORE_PERMISSION_ACL.key -> "false") {
+      withTable("tab1") {
+        sql("CREATE TABLE tab1 (col1 STRING, col2 INT) USING parquet PARTITIONED BY (col2)")
+        sql("INSERT INTO tab1 SELECT 'one', 1")
+        checkAnswer(spark.table("tab1"), Row("one", 1))
+        val part = spark.sessionState.catalog.listPartitions(TableIdentifier("tab1")).head
+        val path = new File(part.location.getPath)
+        sql("TRUNCATE TABLE tab1")
+        // simulate incomplete/unsuccessful truncate
+        assert(path.exists())
+        path.delete()
+        assert(!path.exists())
+        // execute without java.io.FileNotFoundException
+        sql("TRUNCATE TABLE tab1")
+        // partition path should be re-created
+        assert(path.exists())
+      }
+    }
+  }
+
   test("create temporary view with mismatched schema") {
     withTable("tab1") {
       spark.range(10).write.saveAsTable("tab1")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org