You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by fe...@apache.org on 2017/11/10 18:22:47 UTC
spark git commit: [SPARK-22344][SPARKR] clean up install dir if
running test as source package
Repository: spark
Updated Branches:
refs/heads/master 5b41cbf13 -> b70aa9e08
[SPARK-22344][SPARKR] clean up install dir if running test as source package
## What changes were proposed in this pull request?
remove spark if spark downloaded & installed
## How was this patch tested?
manually by building package
Jenkins, AppVeyor
Author: Felix Cheung <fe...@hotmail.com>
Closes #19657 from felixcheung/rinstalldir.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b70aa9e0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b70aa9e0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b70aa9e0
Branch: refs/heads/master
Commit: b70aa9e08b4476746e912c2c2a8b7bdd102305e8
Parents: 5b41cbf
Author: Felix Cheung <fe...@hotmail.com>
Authored: Fri Nov 10 10:22:42 2017 -0800
Committer: Felix Cheung <fe...@apache.org>
Committed: Fri Nov 10 10:22:42 2017 -0800
----------------------------------------------------------------------
R/pkg/R/install.R | 37 ++++++++++++++++++++++++++++++-
R/pkg/R/utils.R | 13 +++++++++++
R/pkg/tests/fulltests/test_utils.R | 25 +++++++++++++++++++++
R/pkg/tests/run-all.R | 4 +++-
R/pkg/vignettes/sparkr-vignettes.Rmd | 6 ++++-
5 files changed, 82 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b70aa9e0/R/pkg/R/install.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/install.R b/R/pkg/R/install.R
index 492dee6..04dc756 100644
--- a/R/pkg/R/install.R
+++ b/R/pkg/R/install.R
@@ -152,6 +152,11 @@ install.spark <- function(hadoopVersion = "2.7", mirrorUrl = NULL,
})
if (!tarExists || overwrite || !success) {
unlink(packageLocalPath)
+ if (success) {
+ # if tar file was not there before (or it was, but we are told to overwrite it),
+ # and untar is successful - set a flag that we have downloaded (and untar) Spark package.
+ assign(".sparkDownloaded", TRUE, envir = .sparkREnv)
+ }
}
if (!success) stop("Extract archive failed.")
message("DONE.")
@@ -266,6 +271,7 @@ hadoopVersionName <- function(hadoopVersion) {
# The implementation refers to appdirs package: https://pypi.python.org/pypi/appdirs and
# adapt to Spark context
+# see also sparkCacheRelPathLength()
sparkCachePath <- function() {
if (is_windows()) {
winAppPath <- Sys.getenv("LOCALAPPDATA", unset = NA)
@@ -282,7 +288,7 @@ sparkCachePath <- function() {
}
} else if (.Platform$OS.type == "unix") {
if (Sys.info()["sysname"] == "Darwin") {
- path <- file.path(Sys.getenv("HOME"), "Library/Caches", "spark")
+ path <- file.path(Sys.getenv("HOME"), "Library", "Caches", "spark")
} else {
path <- file.path(
Sys.getenv("XDG_CACHE_HOME", file.path(Sys.getenv("HOME"), ".cache")), "spark")
@@ -293,6 +299,16 @@ sparkCachePath <- function() {
normalizePath(path, mustWork = FALSE)
}
+# Length of the Spark cache specific relative path segments for each platform
+# eg. "Apache\Spark\Cache" is 3 in Windows, or "spark" is 1 in unix
+# Must match sparkCachePath() exactly.
+sparkCacheRelPathLength <- function() {
+ if (is_windows()) {
+ 3
+ } else {
+ 1
+ }
+}
installInstruction <- function(mode) {
if (mode == "remote") {
@@ -310,3 +326,22 @@ installInstruction <- function(mode) {
stop(paste0("No instruction found for ", mode, " mode."))
}
}
+
+uninstallDownloadedSpark <- function() {
+ # clean up if Spark was downloaded
+ sparkDownloaded <- getOne(".sparkDownloaded",
+ envir = .sparkREnv,
+ inherits = TRUE,
+ ifnotfound = FALSE)
+ sparkDownloadedDir <- Sys.getenv("SPARK_HOME")
+ if (sparkDownloaded && nchar(sparkDownloadedDir) > 0) {
+ unlink(sparkDownloadedDir, recursive = TRUE, force = TRUE)
+
+ dirs <- traverseParentDirs(sparkCachePath(), sparkCacheRelPathLength())
+ lapply(dirs, function(d) {
+ if (length(list.files(d, all.files = TRUE, include.dirs = TRUE, no.. = TRUE)) == 0) {
+ unlink(d, recursive = TRUE, force = TRUE)
+ }
+ })
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/b70aa9e0/R/pkg/R/utils.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/utils.R b/R/pkg/R/utils.R
index fa40992..164cd6d 100644
--- a/R/pkg/R/utils.R
+++ b/R/pkg/R/utils.R
@@ -910,3 +910,16 @@ hadoop_home_set <- function() {
windows_with_hadoop <- function() {
!is_windows() || hadoop_home_set()
}
+
+# get0 not supported before R 3.2.0
+getOne <- function(x, envir, inherits = TRUE, ifnotfound = NULL) {
+ mget(x[1L], envir = envir, inherits = inherits, ifnotfound = list(ifnotfound))[[1L]]
+}
+
+# Returns a vector of parent directories, traversing up count times, starting with a full path
+# eg. traverseParentDirs("/Users/user/Library/Caches/spark/spark2.2", 1) should return
+# this "/Users/user/Library/Caches/spark/spark2.2"
+# and "/Users/user/Library/Caches/spark"
+traverseParentDirs <- function(x, count) {
+ if (dirname(x) == x || count <= 0) x else c(x, Recall(dirname(x), count - 1))
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/b70aa9e0/R/pkg/tests/fulltests/test_utils.R
----------------------------------------------------------------------
diff --git a/R/pkg/tests/fulltests/test_utils.R b/R/pkg/tests/fulltests/test_utils.R
index fb394b8..f0292ab 100644
--- a/R/pkg/tests/fulltests/test_utils.R
+++ b/R/pkg/tests/fulltests/test_utils.R
@@ -228,4 +228,29 @@ test_that("basenameSansExtFromUrl", {
expect_equal(basenameSansExtFromUrl(z), "spark-2.1.0--hive")
})
+test_that("getOne", {
+ dummy <- getOne(".dummyValue", envir = new.env(), ifnotfound = FALSE)
+ expect_equal(dummy, FALSE)
+})
+
+test_that("traverseParentDirs", {
+ if (is_windows()) {
+ # original path is included as-is, otherwise dirname() replaces \\ with / on windows
+ dirs <- traverseParentDirs("c:\\Users\\user\\AppData\\Local\\Apache\\Spark\\Cache\\spark2.2", 3)
+ expect <- c("c:\\Users\\user\\AppData\\Local\\Apache\\Spark\\Cache\\spark2.2",
+ "c:/Users/user/AppData/Local/Apache/Spark/Cache",
+ "c:/Users/user/AppData/Local/Apache/Spark",
+ "c:/Users/user/AppData/Local/Apache")
+ expect_equal(dirs, expect)
+ } else {
+ dirs <- traverseParentDirs("/Users/user/Library/Caches/spark/spark2.2", 1)
+ expect <- c("/Users/user/Library/Caches/spark/spark2.2", "/Users/user/Library/Caches/spark")
+ expect_equal(dirs, expect)
+
+ dirs <- traverseParentDirs("/home/u/.cache/spark/spark2.2", 1)
+ expect <- c("/home/u/.cache/spark/spark2.2", "/home/u/.cache/spark")
+ expect_equal(dirs, expect)
+ }
+})
+
sparkR.session.stop()
http://git-wip-us.apache.org/repos/asf/spark/blob/b70aa9e0/R/pkg/tests/run-all.R
----------------------------------------------------------------------
diff --git a/R/pkg/tests/run-all.R b/R/pkg/tests/run-all.R
index a7f913e..63812ba 100644
--- a/R/pkg/tests/run-all.R
+++ b/R/pkg/tests/run-all.R
@@ -46,7 +46,7 @@ if (identical(Sys.getenv("NOT_CRAN"), "true")) {
tmpDir <- tempdir()
tmpArg <- paste0("-Djava.io.tmpdir=", tmpDir)
sparkRTestConfig <- list(spark.driver.extraJavaOptions = tmpArg,
- spark.executor.extraJavaOptions = tmpArg)
+ spark.executor.extraJavaOptions = tmpArg)
}
test_package("SparkR")
@@ -60,3 +60,5 @@ if (identical(Sys.getenv("NOT_CRAN"), "true")) {
NULL,
"summary")
}
+
+SparkR:::uninstallDownloadedSpark()
http://git-wip-us.apache.org/repos/asf/spark/blob/b70aa9e0/R/pkg/vignettes/sparkr-vignettes.Rmd
----------------------------------------------------------------------
diff --git a/R/pkg/vignettes/sparkr-vignettes.Rmd b/R/pkg/vignettes/sparkr-vignettes.Rmd
index 907bbb3..8c4ea2f 100644
--- a/R/pkg/vignettes/sparkr-vignettes.Rmd
+++ b/R/pkg/vignettes/sparkr-vignettes.Rmd
@@ -37,7 +37,7 @@ opts_hooks$set(eval = function(options) {
options
})
r_tmp_dir <- tempdir()
-tmp_arg <- paste("-Djava.io.tmpdir=", r_tmp_dir, sep = "")
+tmp_arg <- paste0("-Djava.io.tmpdir=", r_tmp_dir)
sparkSessionConfig <- list(spark.driver.extraJavaOptions = tmp_arg,
spark.executor.extraJavaOptions = tmp_arg)
old_java_opt <- Sys.getenv("_JAVA_OPTIONS")
@@ -1183,3 +1183,7 @@ env | map
```{r, echo=FALSE}
sparkR.session.stop()
```
+
+```{r cleanup, include=FALSE}
+SparkR:::uninstallDownloadedSpark()
+```
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org