You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2023/01/10 02:41:01 UTC
[spark] branch master updated: [SPARK-41943][CORE] Use java api to create files and grant permissions
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b22946ed8b5 [SPARK-41943][CORE] Use java api to create files and grant permissions
b22946ed8b5 is described below
commit b22946ed8b5f41648a32a4e0c4c40226141a06a0
Author: smallzhongfeng <zh...@didiglobal.com>
AuthorDate: Tue Jan 10 10:40:44 2023 +0800
[SPARK-41943][CORE] Use java api to create files and grant permissions
### What changes were proposed in this pull request?
For method `createDirWithPermission770`, using java api to create files and grant permissions instead of calling shell commands.
### Why are the changes needed?
Safer and more efficient.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Origin uts.
Closes #39448 from smallzhongfeng/java-api-mkdir.
Authored-by: smallzhongfeng <zh...@didiglobal.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../org/apache/spark/storage/DiskBlockManager.scala | 16 +++++-----------
1 file changed, 5 insertions(+), 11 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
index e29f3fc1b80..a7ed9226c57 100644
--- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
@@ -19,7 +19,7 @@ package org.apache.spark.storage
import java.io.{File, IOException}
import java.nio.file.Files
-import java.nio.file.attribute.PosixFilePermission
+import java.nio.file.attribute.{PosixFilePermission, PosixFilePermissions}
import java.util.UUID
import scala.collection.mutable.HashMap
@@ -301,9 +301,6 @@ private[spark] class DiskBlockManager(
* Create a directory that is writable by the group.
* Grant the permission 770 "rwxrwx---" to the directory so the shuffle server can
* create subdirs/files within the merge folder.
- * TODO: Find out why can't we create a dir using java api with permission 770
- * Files.createDirectories(mergeDir.toPath, PosixFilePermissions.asFileAttribute(
- * PosixFilePermissions.fromString("rwxrwx---")))
*/
def createDirWithPermission770(dirToCreate: File): Unit = {
var attempts = 0
@@ -315,16 +312,13 @@ private[spark] class DiskBlockManager(
throw SparkCoreErrors.failToCreateDirectoryError(dirToCreate.getAbsolutePath, maxAttempts)
}
try {
- val builder = new ProcessBuilder().command(
- "mkdir", "-p", "-m770", dirToCreate.getAbsolutePath)
- val proc = builder.start()
- val exitCode = proc.waitFor()
+ dirToCreate.mkdirs()
+ Files.setPosixFilePermissions(
+ dirToCreate.toPath, PosixFilePermissions.fromString("rwxrwx---"))
if (dirToCreate.exists()) {
created = dirToCreate
}
- logDebug(
- s"Created directory at ${dirToCreate.getAbsolutePath} with permission " +
- s"770 and exitCode $exitCode")
+ logDebug(s"Created directory at ${dirToCreate.getAbsolutePath} with permission 770")
} catch {
case e: SecurityException =>
logWarning(s"Failed to create directory ${dirToCreate.getAbsolutePath} " +
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org