You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sh...@apache.org on 2017/10/30 01:53:52 UTC
spark git commit: [SPARK-22344][SPARKR] Set java.io.tmpdir for SparkR
tests
Repository: spark
Updated Branches:
refs/heads/master 659acf18d -> 1fe27612d
[SPARK-22344][SPARKR] Set java.io.tmpdir for SparkR tests
This PR sets the java.io.tmpdir for CRAN checks and also disables the hsperfdata for the JVM when running CRAN checks. Together this prevents files from being left behind in `/tmp`
## How was this patch tested?
Tested manually on a clean EC2 machine
Author: Shivaram Venkataraman <sh...@cs.berkeley.edu>
Closes #19589 from shivaram/sparkr-tmpdir-clean.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1fe27612
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1fe27612
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1fe27612
Branch: refs/heads/master
Commit: 1fe27612d7bcb8b6478a36bc16ddd4802e4ee2fc
Parents: 659acf1
Author: Shivaram Venkataraman <sh...@cs.berkeley.edu>
Authored: Sun Oct 29 18:53:47 2017 -0700
Committer: Shivaram Venkataraman <sh...@cs.berkeley.edu>
Committed: Sun Oct 29 18:53:47 2017 -0700
----------------------------------------------------------------------
R/pkg/inst/tests/testthat/test_basic.R | 6 ++++--
R/pkg/tests/run-all.R | 9 +++++++++
R/pkg/vignettes/sparkr-vignettes.Rmd | 8 +++++++-
3 files changed, 20 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/1fe27612/R/pkg/inst/tests/testthat/test_basic.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_basic.R b/R/pkg/inst/tests/testthat/test_basic.R
index de47162..823d26f 100644
--- a/R/pkg/inst/tests/testthat/test_basic.R
+++ b/R/pkg/inst/tests/testthat/test_basic.R
@@ -18,7 +18,8 @@
context("basic tests for CRAN")
test_that("create DataFrame from list or data.frame", {
- sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE)
+ sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE,
+ sparkConfig = sparkRTestConfig)
i <- 4
df <- createDataFrame(data.frame(dummy = 1:i))
@@ -49,7 +50,8 @@ test_that("create DataFrame from list or data.frame", {
})
test_that("spark.glm and predict", {
- sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE)
+ sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE,
+ sparkConfig = sparkRTestConfig)
training <- suppressWarnings(createDataFrame(iris))
# gaussian family
http://git-wip-us.apache.org/repos/asf/spark/blob/1fe27612/R/pkg/tests/run-all.R
----------------------------------------------------------------------
diff --git a/R/pkg/tests/run-all.R b/R/pkg/tests/run-all.R
index a1834a2..a7f913e 100644
--- a/R/pkg/tests/run-all.R
+++ b/R/pkg/tests/run-all.R
@@ -36,8 +36,17 @@ invisible(lapply(sparkRWhitelistSQLDirs,
sparkRFilesBefore <- list.files(path = sparkRDir, all.files = TRUE)
sparkRTestMaster <- "local[1]"
+sparkRTestConfig <- list()
if (identical(Sys.getenv("NOT_CRAN"), "true")) {
sparkRTestMaster <- ""
+} else {
+ # Disable hsperfdata on CRAN
+ old_java_opt <- Sys.getenv("_JAVA_OPTIONS")
+ Sys.setenv("_JAVA_OPTIONS" = paste("-XX:-UsePerfData", old_java_opt))
+ tmpDir <- tempdir()
+ tmpArg <- paste0("-Djava.io.tmpdir=", tmpDir)
+ sparkRTestConfig <- list(spark.driver.extraJavaOptions = tmpArg,
+ spark.executor.extraJavaOptions = tmpArg)
}
test_package("SparkR")
http://git-wip-us.apache.org/repos/asf/spark/blob/1fe27612/R/pkg/vignettes/sparkr-vignettes.Rmd
----------------------------------------------------------------------
diff --git a/R/pkg/vignettes/sparkr-vignettes.Rmd b/R/pkg/vignettes/sparkr-vignettes.Rmd
index caeae72..907bbb3 100644
--- a/R/pkg/vignettes/sparkr-vignettes.Rmd
+++ b/R/pkg/vignettes/sparkr-vignettes.Rmd
@@ -36,6 +36,12 @@ opts_hooks$set(eval = function(options) {
}
options
})
+r_tmp_dir <- tempdir()
+tmp_arg <- paste("-Djava.io.tmpdir=", r_tmp_dir, sep = "")
+sparkSessionConfig <- list(spark.driver.extraJavaOptions = tmp_arg,
+ spark.executor.extraJavaOptions = tmp_arg)
+old_java_opt <- Sys.getenv("_JAVA_OPTIONS")
+Sys.setenv("_JAVA_OPTIONS" = paste("-XX:-UsePerfData", old_java_opt, sep = " "))
```
## Overview
@@ -57,7 +63,7 @@ We use default settings in which it runs in local mode. It auto downloads Spark
```{r, include=FALSE}
install.spark()
-sparkR.session(master = "local[1]")
+sparkR.session(master = "local[1]", sparkConfig = sparkSessionConfig, enableHiveSupport = FALSE)
```
```{r, eval=FALSE}
sparkR.session()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org