You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2018/09/26 15:37:21 UTC
spark git commit: [SPARK-25509][CORE] Windows doesn't support POSIX
permissions
Repository: spark
Updated Branches:
refs/heads/master cf5c9c4b5 -> a2ac5a72c
[SPARK-25509][CORE] Windows doesn't support POSIX permissions
## What changes were proposed in this pull request?
SHS V2 cannot enabled in Windows, because windows doesn't support POSIX permission.
## How was this patch tested?
test case fails in windows without this fix.
org.apache.spark.deploy.history.HistoryServerDiskManagerSuite test("leasing space")
SHS V2 cannot run successfully in Windows without this fix.
java.lang.UnsupportedOperationException: 'posix:permissions' not supported as initial attribute
at sun.nio.fs.WindowsSecurityDescriptor.fromAttribute(WindowsSecurityDescriptor.java:358)
Closes #22520 from jianjianjiao/FixWindowsPermssionsIssue.
Authored-by: Rong Tang <ro...@microsoft.com>
Signed-off-by: Sean Owen <se...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a2ac5a72
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a2ac5a72
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a2ac5a72
Branch: refs/heads/master
Commit: a2ac5a72ccd2b14c8492d4a6da9e8b30f0f3c9b4
Parents: cf5c9c4
Author: Rong Tang <ro...@microsoft.com>
Authored: Wed Sep 26 10:37:17 2018 -0500
Committer: Sean Owen <se...@databricks.com>
Committed: Wed Sep 26 10:37:17 2018 -0500
----------------------------------------------------------------------
.../org/apache/spark/deploy/history/FsHistoryProvider.scala | 6 ++----
.../spark/deploy/history/HistoryServerDiskManager.scala | 7 ++-----
2 files changed, 4 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a2ac5a72/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index 44d2390..c23a659 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -19,7 +19,6 @@ package org.apache.spark.deploy.history
import java.io.{File, FileNotFoundException, IOException}
import java.nio.file.Files
-import java.nio.file.attribute.PosixFilePermissions
import java.util.{Date, ServiceLoader}
import java.util.concurrent.{ConcurrentHashMap, ExecutorService, Future, TimeUnit}
import java.util.zip.{ZipEntry, ZipOutputStream}
@@ -133,9 +132,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
// Visible for testing.
private[history] val listing: KVStore = storePath.map { path =>
- val perms = PosixFilePermissions.fromString("rwx------")
- val dbPath = Files.createDirectories(new File(path, "listing.ldb").toPath(),
- PosixFilePermissions.asFileAttribute(perms)).toFile()
+ val dbPath = Files.createDirectories(new File(path, "listing.ldb").toPath()).toFile()
+ Utils.chmod700(dbPath)
val metadata = new FsHistoryProviderMetadata(CURRENT_LISTING_VERSION,
AppStatusStore.CURRENT_VERSION, logDir.toString())
http://git-wip-us.apache.org/repos/asf/spark/blob/a2ac5a72/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
index c03a360..ad0dd23 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
@@ -18,8 +18,6 @@
package org.apache.spark.deploy.history
import java.io.File
-import java.nio.file.Files
-import java.nio.file.attribute.PosixFilePermissions
import java.util.concurrent.atomic.AtomicLong
import scala.collection.JavaConverters._
@@ -107,9 +105,8 @@ private class HistoryServerDiskManager(
val needed = approximateSize(eventLogSize, isCompressed)
makeRoom(needed)
- val perms = PosixFilePermissions.fromString("rwx------")
- val tmp = Files.createTempDirectory(tmpStoreDir.toPath(), "appstore",
- PosixFilePermissions.asFileAttribute(perms)).toFile()
+ val tmp = Utils.createTempDir(tmpStoreDir.getPath(), "appstore")
+ Utils.chmod700(tmp)
updateUsage(needed)
val current = currentUsage.get()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org