You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/08/05 04:21:32 UTC
(spark) branch master updated: [SPARK-53121][SQL] Use `deleteRecursively` instead of `FileUtils.forceDelete`
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new faa6723125d0 [SPARK-53121][SQL] Use `deleteRecursively` instead of `FileUtils.forceDelete`
faa6723125d0 is described below
commit faa6723125d07440b0090a18b713c2723fa2ffcd
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Mon Aug 4 21:21:20 2025 -0700
[SPARK-53121][SQL] Use `deleteRecursively` instead of `FileUtils.forceDelete`
### What changes were proposed in this pull request?
This PR aims to use `deleteRecursively` instead of `FileUtils.forceDelete`.
```scala
- FileUtils.forceDelete(dir)
+ Utils.deleteRecursively(dir)
```
```scala
- FileUtils.forceDelete(dir)
+ JavaUtils.deleteRecursively(dir)
```
### Why are the changes needed?
Apache Spark provides `Utils.deleteRecursively` which contains two versions, `deleteRecursivelyUsingUnixNative` and `deleteRecursivelyUsingJavaIO`. We had better take advantage of Spark's built-in optimized one because it's more optimized than `commons-io`.
https://github.com/apache/spark/blob/7c6693ddec00c32d50d254817a7f09a993d71017/core/src/main/scala/org/apache/spark/util/Utils.scala#L1047-L1052
https://github.com/apache/spark/blob/7c6693ddec00c32d50d254817a7f09a993d71017/common/utils/src/main/scala/org/apache/spark/util/SparkFileUtils.scala#L120-L122
https://github.com/apache/spark/blob/7c6693ddec00c32d50d254817a7f09a993d71017/common/utils/src/main/java/org/apache/spark/network/util/JavaUtils.java#L105-L125
### Does this PR introduce _any_ user-facing change?
No behavior change.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #51840 from dongjoon-hyun/SPARK-53121.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
dev/checkstyle.xml | 6 +++++-
scalastyle-config.xml | 7 ++++++-
.../java/org/apache/hive/service/cli/session/HiveSessionImpl.java | 6 +++---
.../java/org/apache/hive/service/cli/session/SessionManager.java | 2 +-
.../org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala | 5 ++---
5 files changed, 17 insertions(+), 9 deletions(-)
diff --git a/dev/checkstyle.xml b/dev/checkstyle.xml
index e1ca3e87a233..e6d06362bba0 100644
--- a/dev/checkstyle.xml
+++ b/dev/checkstyle.xml
@@ -209,7 +209,11 @@
</module>
<module name="RegexpSinglelineJava">
<property name="format" value="FileUtils\.deleteDirectory"/>
- <property name="message" value="Use deleteRecursively of SparkFileUtils or Utils instead." />
+ <property name="message" value="Use deleteRecursively of JavaUtils/SparkFileUtils/Utils instead." />
+ </module>
+ <module name="RegexpSinglelineJava">
+ <property name="format" value="FileUtils\.forceDelete\("/>
+ <property name="message" value="Use deleteRecursively of JavaUtils/SparkFileUtils/Utils instead." />
</module>
<module name="RegexpSinglelineJava">
<property name="format" value="FileUtils\.deleteQuietly"/>
diff --git a/scalastyle-config.xml b/scalastyle-config.xml
index 75fe47a2495a..c9583ee19583 100644
--- a/scalastyle-config.xml
+++ b/scalastyle-config.xml
@@ -309,7 +309,12 @@ This file is divided into 3 sections:
<check customId="deleteRecursively" level="error" class="org.scalastyle.file.RegexChecker" enabled="true">
<parameters><parameter name="regex">\bFileUtils\.deleteDirectory\b</parameter></parameters>
- <customMessage>Use deleteRecursively of SparkFileUtils or Utils</customMessage>
+ <customMessage>Use deleteRecursively of JavaUtils/SparkFileUtils/Utils</customMessage>
+ </check>
+
+ <check customId="forceDelete" level="error" class="org.scalastyle.file.RegexChecker" enabled="true">
+ <parameters><parameter name="regex">\bFileUtils\.forceDelete\b</parameter></parameters>
+ <customMessage>Use deleteRecursively of JavaUtils/SparkFileUtils/Utils</customMessage>
</check>
<check customId="deleteQuietly" level="error" class="org.scalastyle.file.RegexChecker" enabled="true">
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 084ca486b76b..8ec7a64db8ec 100644
--- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -28,7 +28,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hive.common.cli.HiveFileProcessor;
import org.apache.hadoop.hive.common.cli.IHiveFileProcessor;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -74,6 +73,7 @@ import org.apache.spark.internal.SparkLogger;
import org.apache.spark.internal.SparkLoggerFactory;
import org.apache.spark.internal.LogKeys;
import org.apache.spark.internal.MDC;
+import org.apache.spark.network.util.JavaUtils;
import org.apache.spark.util.Utils;
import org.apache.spark.util.SparkStringUtils;
@@ -714,7 +714,7 @@ public class HiveSessionImpl implements HiveSession {
} else {
for (File file : fileAry) {
try {
- FileUtils.forceDelete(file);
+ JavaUtils.deleteRecursively(file);
} catch (Exception e) {
LOG.error("Failed to cleanup pipeout file: {}", e, MDC.of(LogKeys.PATH$.MODULE$, file));
}
@@ -725,7 +725,7 @@ public class HiveSessionImpl implements HiveSession {
private void cleanupSessionLogDir() {
if (isOperationLogEnabled) {
try {
- FileUtils.forceDelete(sessionLogDir);
+ JavaUtils.deleteRecursively(sessionLogDir);
} catch (Exception e) {
LOG.error("Failed to cleanup session log dir: {}", e,
MDC.of(LogKeys.SESSION_HANDLE$.MODULE$, sessionHandle));
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/SessionManager.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
index 83d4c7a3622f..27c98683d85e 100644
--- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
@@ -231,7 +231,7 @@ public class SessionManager extends CompositeService {
private void cleanupLoggingRootDir() {
if (isOperationLogEnabled) {
try {
- FileUtils.forceDelete(operationLogRootDir);
+ Utils.deleteRecursively(operationLogRootDir);
} catch (Exception e) {
LOG.warn("Failed to cleanup root dir of HS2 logging: {}", e,
MDC.of(LogKeys.PATH$.MODULE$, operationLogRootDir.getAbsolutePath()));
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala
index d0bfa28f9f55..c003c8a3e719 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala
@@ -17,7 +17,6 @@
package org.apache.spark.sql.hive.thriftserver
-import org.apache.commons.io.FileUtils
import test.custom.listener.{DummyQueryExecutionListener, DummyStreamingQueryListener}
import org.apache.spark.SparkFunSuite
@@ -31,7 +30,7 @@ import org.apache.spark.util.Utils
class SparkSQLEnvSuite extends SparkFunSuite {
test("SPARK-29604 external listeners should be initialized with Spark classloader") {
val metastorePath = Utils.createTempDir("spark_derby")
- FileUtils.forceDelete(metastorePath)
+ Utils.deleteRecursively(metastorePath)
val jdbcUrl = s"jdbc:derby:;databaseName=$metastorePath;create=true"
@@ -61,7 +60,7 @@ class SparkSQLEnvSuite extends SparkFunSuite {
} finally {
SparkSQLEnv.stop()
if (metastorePath.exists()) {
- FileUtils.forceDelete(metastorePath)
+ Utils.deleteRecursively(metastorePath)
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org