You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2016/02/09 09:43:49 UTC
spark git commit: [SPARK-13176][CORE] Use native file linking instead
of external process ln
Repository: spark
Updated Branches:
refs/heads/master 3708d13f1 -> f9307d8fc
[SPARK-13176][CORE] Use native file linking instead of external process ln
Since Spark requires at least JRE 1.7, it is safe to use built-in java.nio.Files.
Author: Jakob Odersky <ja...@odersky.com>
Closes #11098 from jodersky/SPARK-13176.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f9307d8f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f9307d8f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f9307d8f
Branch: refs/heads/master
Commit: f9307d8fc5223b4c5be07e3dc691a327f3bbfa7f
Parents: 3708d13
Author: Jakob Odersky <ja...@odersky.com>
Authored: Tue Feb 9 08:43:46 2016 +0000
Committer: Sean Owen <so...@cloudera.com>
Committed: Tue Feb 9 08:43:46 2016 +0000
----------------------------------------------------------------------
.../scala/org/apache/spark/util/Utils.scala | 27 ++++++--------------
1 file changed, 8 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/f9307d8f/core/src/main/scala/org/apache/spark/util/Utils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 9ecbffb..e0c9bf0 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -22,6 +22,7 @@ import java.lang.management.ManagementFactory
import java.net._
import java.nio.ByteBuffer
import java.nio.channels.Channels
+import java.nio.file.Files
import java.util.{Locale, Properties, Random, UUID}
import java.util.concurrent._
import javax.net.ssl.HttpsURLConnection
@@ -34,7 +35,7 @@ import scala.reflect.ClassTag
import scala.util.Try
import scala.util.control.{ControlThrowable, NonFatal}
-import com.google.common.io.{ByteStreams, Files}
+import com.google.common.io.{ByteStreams, Files => GFiles}
import com.google.common.net.InetAddresses
import org.apache.commons.lang3.SystemUtils
import org.apache.hadoop.conf.Configuration
@@ -516,7 +517,7 @@ private[spark] object Utils extends Logging {
// The file does not exist in the target directory. Copy or move it there.
if (removeSourceFile) {
- Files.move(sourceFile, destFile)
+ Files.move(sourceFile.toPath, destFile.toPath)
} else {
logInfo(s"Copying ${sourceFile.getAbsolutePath} to ${destFile.getAbsolutePath}")
copyRecursive(sourceFile, destFile)
@@ -534,7 +535,7 @@ private[spark] object Utils extends Logging {
case (f1, f2) => filesEqualRecursive(f1, f2)
}
} else if (file1.isFile && file2.isFile) {
- Files.equal(file1, file2)
+ GFiles.equal(file1, file2)
} else {
false
}
@@ -548,7 +549,7 @@ private[spark] object Utils extends Logging {
val subfiles = source.listFiles()
subfiles.foreach(f => copyRecursive(f, new File(dest, f.getName)))
} else {
- Files.copy(source, dest)
+ Files.copy(source.toPath, dest.toPath)
}
}
@@ -1596,30 +1597,18 @@ private[spark] object Utils extends Logging {
}
/**
- * Creates a symlink. Note jdk1.7 has Files.createSymbolicLink but not used here
- * for jdk1.6 support. Supports windows by doing copy, everything else uses "ln -sf".
+ * Creates a symlink.
* @param src absolute path to the source
* @param dst relative path for the destination
*/
- def symlink(src: File, dst: File) {
+ def symlink(src: File, dst: File): Unit = {
if (!src.isAbsolute()) {
throw new IOException("Source must be absolute")
}
if (dst.isAbsolute()) {
throw new IOException("Destination must be relative")
}
- var cmdSuffix = ""
- val linkCmd = if (isWindows) {
- // refer to http://technet.microsoft.com/en-us/library/cc771254.aspx
- cmdSuffix = " /s /e /k /h /y /i"
- "cmd /c xcopy "
- } else {
- cmdSuffix = ""
- "ln -sf "
- }
- import scala.sys.process._
- (linkCmd + src.getAbsolutePath() + " " + dst.getPath() + cmdSuffix) lines_!
- ProcessLogger(line => logInfo(line))
+ Files.createSymbolicLink(dst.toPath, src.toPath)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org